2024-12-04 20:27:29,783 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 20:27:29,795 main DEBUG Took 0.010505 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 20:27:29,795 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 20:27:29,796 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 20:27:29,797 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 20:27:29,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,806 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 20:27:29,820 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,822 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,823 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,825 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,826 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,828 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,829 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,830 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,830 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,831 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,832 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,833 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:27:29,834 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,834 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 20:27:29,836 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:27:29,840 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 20:27:29,843 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 20:27:29,843 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 20:27:29,845 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 20:27:29,845 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 20:27:29,855 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 20:27:29,858 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 20:27:29,860 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 20:27:29,861 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 20:27:29,861 main DEBUG createAppenders(={Console}) 2024-12-04 20:27:29,862 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-04 20:27:29,863 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 20:27:29,863 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-04 20:27:29,864 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 20:27:29,864 main DEBUG OutputStream closed 2024-12-04 20:27:29,865 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 20:27:29,865 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 20:27:29,865 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-04 20:27:29,937 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 20:27:29,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 20:27:29,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 20:27:29,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 20:27:29,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 20:27:29,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 20:27:29,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 20:27:29,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 20:27:29,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 20:27:29,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 20:27:29,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 20:27:29,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 20:27:29,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 20:27:29,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 20:27:29,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 20:27:29,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 20:27:29,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 20:27:29,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 20:27:29,950 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 20:27:29,950 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-04 20:27:29,951 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 20:27:29,952 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-04T20:27:30,199 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983 2024-12-04 20:27:30,202 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 20:27:30,203 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T20:27:30,211 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-04T20:27:30,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:27:30,244 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4, deleteOnExit=true 2024-12-04T20:27:30,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:27:30,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/test.cache.data in system properties and HBase conf 2024-12-04T20:27:30,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:27:30,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:27:30,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:27:30,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:27:30,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:27:30,333 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T20:27:30,415 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:27:30,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:27:30,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:27:30,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:27:30,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:27:30,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:27:30,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:27:30,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:27:30,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:27:30,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:27:30,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:27:30,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:27:30,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:27:30,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:27:30,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:27:31,143 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T20:27:31,219 INFO [Time-limited test {}] log.Log(170): Logging initialized @2172ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T20:27:31,288 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:27:31,346 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:27:31,389 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:27:31,389 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:27:31,392 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:27:31,411 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:27:31,415 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:27:31,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:27:31,612 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/java.io.tmpdir/jetty-localhost-38917-hadoop-hdfs-3_4_1-tests_jar-_-any-13695631992055849916/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:27:31,620 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38917} 2024-12-04T20:27:31,620 INFO [Time-limited test {}] server.Server(415): Started @2574ms 2024-12-04T20:27:31,990 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:27:31,996 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:27:31,997 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:27:31,998 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:27:31,998 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:27:31,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:27:31,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:27:32,097 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/java.io.tmpdir/jetty-localhost-44321-hadoop-hdfs-3_4_1-tests_jar-_-any-6251692804882378311/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:27:32,098 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:44321} 2024-12-04T20:27:32,099 INFO [Time-limited test {}] server.Server(415): Started @3052ms 2024-12-04T20:27:32,147 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:27:32,590 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4/data/data1/current/BP-994321494-172.17.0.2-1733344050940/current, will proceed with Du for space computation calculation, 2024-12-04T20:27:32,591 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4/data/data2/current/BP-994321494-172.17.0.2-1733344050940/current, will proceed with Du for space computation calculation, 2024-12-04T20:27:32,644 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:27:32,698 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6228735428876ea4 with lease ID 0x6b6cd25b54622e23: Processing first storage report for DS-bee32ce1-fb5f-4989-b327-f15dc7c1a63c from datanode DatanodeRegistration(127.0.0.1:42649, datanodeUuid=4ea7031d-452d-42d8-9831-975d4a4a5e30, infoPort=41109, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1515410867;c=1733344050940) 2024-12-04T20:27:32,699 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6228735428876ea4 with lease ID 0x6b6cd25b54622e23: from storage DS-bee32ce1-fb5f-4989-b327-f15dc7c1a63c node DatanodeRegistration(127.0.0.1:42649, datanodeUuid=4ea7031d-452d-42d8-9831-975d4a4a5e30, infoPort=41109, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1515410867;c=1733344050940), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T20:27:32,700 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6228735428876ea4 with lease ID 0x6b6cd25b54622e23: Processing first storage report for DS-3aab4363-bba5-4b34-93fb-ea8e0ba755d1 from datanode DatanodeRegistration(127.0.0.1:42649, datanodeUuid=4ea7031d-452d-42d8-9831-975d4a4a5e30, infoPort=41109, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1515410867;c=1733344050940) 2024-12-04T20:27:32,700 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6228735428876ea4 with lease ID 0x6b6cd25b54622e23: from storage DS-3aab4363-bba5-4b34-93fb-ea8e0ba755d1 node DatanodeRegistration(127.0.0.1:42649, datanodeUuid=4ea7031d-452d-42d8-9831-975d4a4a5e30, infoPort=41109, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1515410867;c=1733344050940), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T20:27:32,790 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983 2024-12-04T20:27:32,875 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4/zookeeper_0, clientPort=57628, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:27:32,887 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57628 2024-12-04T20:27:32,898 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:27:32,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:27:33,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:27:33,545 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 with version=8 2024-12-04T20:27:33,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/hbase-staging 2024-12-04T20:27:33,621 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T20:27:33,818 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:27:33,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:27:33,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:27:33,831 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:27:33,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:27:33,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:27:33,967 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:27:34,030 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T20:27:34,038 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T20:27:34,041 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:27:34,064 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 92365 (auto-detected) 2024-12-04T20:27:34,065 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T20:27:34,083 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43507 2024-12-04T20:27:34,105 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43507 connecting to ZooKeeper ensemble=127.0.0.1:57628 2024-12-04T20:27:34,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435070x0, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:27:34,143 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43507-0x100a6eaa2e80000 connected 2024-12-04T20:27:34,172 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:27:34,174 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:27:34,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:27:34,189 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7, hbase.cluster.distributed=false 2024-12-04T20:27:34,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:27:34,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43507 2024-12-04T20:27:34,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43507 2024-12-04T20:27:34,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43507 2024-12-04T20:27:34,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43507 2024-12-04T20:27:34,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43507 2024-12-04T20:27:34,333 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:27:34,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:27:34,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:27:34,336 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:27:34,336 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:27:34,336 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:27:34,339 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:27:34,340 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:27:34,341 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45443 2024-12-04T20:27:34,343 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45443 connecting to ZooKeeper ensemble=127.0.0.1:57628 2024-12-04T20:27:34,344 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:27:34,348 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:27:34,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454430x0, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:27:34,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:454430x0, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:27:34,356 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45443-0x100a6eaa2e80001 connected 2024-12-04T20:27:34,360 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:27:34,369 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:27:34,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:27:34,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:27:34,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45443 2024-12-04T20:27:34,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45443 2024-12-04T20:27:34,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45443 2024-12-04T20:27:34,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45443 2024-12-04T20:27:34,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45443 2024-12-04T20:27:34,405 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:43507 2024-12-04T20:27:34,406 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,43507,1733344053664 2024-12-04T20:27:34,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:27:34,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:27:34,413 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,43507,1733344053664 2024-12-04T20:27:34,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:34,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:27:34,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:34,433 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:27:34,434 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,43507,1733344053664 from backup master directory 2024-12-04T20:27:34,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:27:34,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,43507,1733344053664 2024-12-04T20:27:34,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:27:34,437 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:27:34,437 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,43507,1733344053664 2024-12-04T20:27:34,439 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T20:27:34,441 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T20:27:34,497 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/hbase.id] with ID: 1486f5e1-bfc3-483f-a551-5e4afcecaf44 2024-12-04T20:27:34,497 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/.tmp/hbase.id 2024-12-04T20:27:34,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:27:34,913 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/.tmp/hbase.id]:[hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/hbase.id] 2024-12-04T20:27:34,955 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:27:34,959 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:27:34,977 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-04T20:27:34,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:34,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:34,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:27:35,409 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:27:35,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:27:35,424 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:35,428 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:27:35,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:27:35,469 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store 2024-12-04T20:27:35,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:27:35,494 INFO [master/c2ef38372881:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T20:27:35,497 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:27:35,498 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:27:35,498 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:27:35,498 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:27:35,499 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:27:35,499 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:27:35,500 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:27:35,501 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733344055498Disabling compacts and flushes for region at 1733344055498Disabling writes for close at 1733344055499 (+1 ms)Writing region close event to WAL at 1733344055500 (+1 ms)Closed at 1733344055500 2024-12-04T20:27:35,503 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/.initializing 2024-12-04T20:27:35,503 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/WALs/c2ef38372881,43507,1733344053664 2024-12-04T20:27:35,513 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:27:35,528 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C43507%2C1733344053664, suffix=, logDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/WALs/c2ef38372881,43507,1733344053664, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/oldWALs, maxLogs=10 2024-12-04T20:27:35,559 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/WALs/c2ef38372881,43507,1733344053664/c2ef38372881%2C43507%2C1733344053664.1733344055534, exclude list is [], retry=0 2024-12-04T20:27:35,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42649,DS-bee32ce1-fb5f-4989-b327-f15dc7c1a63c,DISK] 2024-12-04T20:27:35,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T20:27:35,615 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/WALs/c2ef38372881,43507,1733344053664/c2ef38372881%2C43507%2C1733344053664.1733344055534 2024-12-04T20:27:35,616 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41109:41109)] 2024-12-04T20:27:35,617 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:27:35,617 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:27:35,621 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,622 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:27:35,686 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:35,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:35,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:27:35,693 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:35,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:27:35,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:27:35,698 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:35,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:27:35,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,702 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:27:35,702 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:35,703 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:27:35,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,707 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,708 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,712 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,713 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,716 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:27:35,719 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:27:35,723 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:27:35,724 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75041538, jitterRate=0.11820605397224426}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:27:35,729 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733344055638Initializing all the Stores at 1733344055639 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733344055640 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344055640Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344055641 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344055641Cleaning up temporary data from old regions at 1733344055713 (+72 ms)Region opened successfully at 1733344055728 (+15 ms) 2024-12-04T20:27:35,730 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:27:35,758 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb85e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:27:35,784 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:27:35,793 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:27:35,793 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:27:35,796 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:27:35,797 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T20:27:35,801 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-04T20:27:35,801 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:27:35,822 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:27:35,830 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:27:35,831 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:27:35,833 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:27:35,835 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:27:35,836 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:27:35,838 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:27:35,841 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:27:35,842 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:27:35,844 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:27:35,845 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:27:35,861 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:27:35,862 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:27:35,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:27:35,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:27:35,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:35,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:35,869 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,43507,1733344053664, sessionid=0x100a6eaa2e80000, setting cluster-up flag (Was=false) 2024-12-04T20:27:35,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:35,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:35,882 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:27:35,884 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,43507,1733344053664 2024-12-04T20:27:35,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:35,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:35,893 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:27:35,894 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,43507,1733344053664 2024-12-04T20:27:35,900 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:27:35,972 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:27:35,984 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:27:35,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:27:35,994 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(746): ClusterId : 1486f5e1-bfc3-483f-a551-5e4afcecaf44 2024-12-04T20:27:35,996 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:27:36,001 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:27:36,001 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:27:35,999 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,43507,1733344053664 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:27:36,003 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:27:36,004 DEBUG [RS:0;c2ef38372881:45443 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3282eecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:27:36,007 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:27:36,007 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:27:36,008 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:27:36,008 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:27:36,008 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:27:36,008 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,008 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:27:36,008 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,017 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733344086017 2024-12-04T20:27:36,017 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:27:36,018 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:27:36,019 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:27:36,021 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:27:36,022 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:45443 2024-12-04T20:27:36,023 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:36,023 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:27:36,025 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:27:36,025 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:27:36,026 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:27:36,026 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:27:36,026 INFO [RS:0;c2ef38372881:45443 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:27:36,026 INFO [RS:0;c2ef38372881:45443 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:27:36,026 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:27:36,029 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,43507,1733344053664 with port=45443, startcode=1733344054291 2024-12-04T20:27:36,029 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,033 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:27:36,035 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:27:36,036 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:27:36,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:27:36,042 DEBUG [RS:0;c2ef38372881:45443 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:27:36,043 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:27:36,044 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:27:36,049 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733344056045,5,FailOnTimeoutGroup] 2024-12-04T20:27:36,050 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733344056050,5,FailOnTimeoutGroup] 2024-12-04T20:27:36,051 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,051 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:27:36,052 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,052 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,080 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35729, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:27:36,088 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43507 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,45443,1733344054291 2024-12-04T20:27:36,091 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43507 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,45443,1733344054291 2024-12-04T20:27:36,108 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:27:36,109 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41363 2024-12-04T20:27:36,109 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:27:36,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:27:36,113 DEBUG [RS:0;c2ef38372881:45443 {}] zookeeper.ZKUtil(111): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,45443,1733344054291 2024-12-04T20:27:36,113 WARN [RS:0;c2ef38372881:45443 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:27:36,113 INFO [RS:0;c2ef38372881:45443 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:27:36,114 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/WALs/c2ef38372881,45443,1733344054291 2024-12-04T20:27:36,115 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,45443,1733344054291] 2024-12-04T20:27:36,141 INFO [RS:0;c2ef38372881:45443 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:27:36,158 INFO [RS:0;c2ef38372881:45443 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:27:36,163 INFO [RS:0;c2ef38372881:45443 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:27:36,164 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,165 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:27:36,171 INFO [RS:0;c2ef38372881:45443 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:27:36,173 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,173 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,173 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,174 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,174 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,174 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,174 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:27:36,174 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,175 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,175 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,175 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,175 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,175 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:27:36,176 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:27:36,176 DEBUG [RS:0;c2ef38372881:45443 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:27:36,177 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,177 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,177 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,177 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,178 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,178 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,45443,1733344054291-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:27:36,193 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:27:36,195 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,45443,1733344054291-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,196 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,196 INFO [RS:0;c2ef38372881:45443 {}] regionserver.Replication(171): c2ef38372881,45443,1733344054291 started 2024-12-04T20:27:36,213 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:36,213 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,45443,1733344054291, RpcServer on c2ef38372881/172.17.0.2:45443, sessionid=0x100a6eaa2e80001 2024-12-04T20:27:36,214 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:27:36,214 DEBUG [RS:0;c2ef38372881:45443 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,45443,1733344054291 2024-12-04T20:27:36,214 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,45443,1733344054291' 2024-12-04T20:27:36,214 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:27:36,215 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:27:36,216 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:27:36,216 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:27:36,216 DEBUG [RS:0;c2ef38372881:45443 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,45443,1733344054291 2024-12-04T20:27:36,216 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,45443,1733344054291' 2024-12-04T20:27:36,216 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:27:36,217 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:27:36,217 DEBUG [RS:0;c2ef38372881:45443 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:27:36,217 INFO [RS:0;c2ef38372881:45443 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:27:36,217 INFO [RS:0;c2ef38372881:45443 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:27:36,321 INFO [RS:0;c2ef38372881:45443 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:27:36,324 INFO [RS:0;c2ef38372881:45443 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C45443%2C1733344054291, suffix=, logDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/WALs/c2ef38372881,45443,1733344054291, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/oldWALs, maxLogs=32 2024-12-04T20:27:36,338 DEBUG [RS:0;c2ef38372881:45443 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/WALs/c2ef38372881,45443,1733344054291/c2ef38372881%2C45443%2C1733344054291.1733344056326, exclude list is [], retry=0 2024-12-04T20:27:36,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42649,DS-bee32ce1-fb5f-4989-b327-f15dc7c1a63c,DISK] 2024-12-04T20:27:36,344 INFO [RS:0;c2ef38372881:45443 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/WALs/c2ef38372881,45443,1733344054291/c2ef38372881%2C45443%2C1733344054291.1733344056326 2024-12-04T20:27:36,345 DEBUG [RS:0;c2ef38372881:45443 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41109:41109)] 2024-12-04T20:27:36,445 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:27:36,446 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:27:36,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741833_1009 (size=32) 2024-12-04T20:27:36,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:27:36,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:27:36,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:27:36,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:36,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:36,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:27:36,870 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:27:36,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:36,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:36,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:27:36,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:27:36,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:36,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:36,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:27:36,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:27:36,878 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:36,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:36,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:27:36,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740 2024-12-04T20:27:36,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740 2024-12-04T20:27:36,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:27:36,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:27:36,885 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:27:36,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:27:36,890 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:27:36,891 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69067355, jitterRate=0.02918379008769989}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:27:36,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733344056860Initializing all the Stores at 1733344056862 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733344056862Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733344056862Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344056862Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733344056862Cleaning up temporary data from old regions at 1733344056884 (+22 ms)Region opened successfully at 1733344056893 (+9 ms) 2024-12-04T20:27:36,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:27:36,893 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:27:36,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:27:36,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:27:36,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:27:36,895 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:27:36,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733344056893Disabling compacts and flushes for region at 1733344056893Disabling writes for close at 1733344056894 (+1 ms)Writing region close event to WAL at 1733344056895 (+1 ms)Closed at 1733344056895 2024-12-04T20:27:36,899 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:27:36,900 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:27:36,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:27:36,914 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:27:36,917 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:27:37,071 DEBUG [c2ef38372881:43507 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:27:37,082 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:27:37,087 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,45443,1733344054291, state=OPENING 2024-12-04T20:27:37,091 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:27:37,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:37,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:27:37,093 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:27:37,093 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:27:37,095 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:27:37,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:27:37,269 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:27:37,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44045, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:27:37,285 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:27:37,285 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:27:37,286 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T20:27:37,289 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C45443%2C1733344054291.meta, suffix=.meta, logDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/WALs/c2ef38372881,45443,1733344054291, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/oldWALs, maxLogs=32 2024-12-04T20:27:37,301 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/WALs/c2ef38372881,45443,1733344054291/c2ef38372881%2C45443%2C1733344054291.meta.1733344057290.meta, exclude list is [], retry=0 2024-12-04T20:27:37,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42649,DS-bee32ce1-fb5f-4989-b327-f15dc7c1a63c,DISK] 2024-12-04T20:27:37,308 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/WALs/c2ef38372881,45443,1733344054291/c2ef38372881%2C45443%2C1733344054291.meta.1733344057290.meta 2024-12-04T20:27:37,309 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41109:41109)] 2024-12-04T20:27:37,309 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:27:37,311 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:27:37,313 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:27:37,317 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:27:37,321 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:27:37,322 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:27:37,322 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:27:37,322 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:27:37,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:27:37,327 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:27:37,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:37,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:37,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:27:37,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:27:37,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:37,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:37,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:27:37,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:27:37,334 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:37,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:37,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:27:37,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:27:37,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:37,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:27:37,338 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:27:37,340 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740 2024-12-04T20:27:37,342 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740 2024-12-04T20:27:37,345 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:27:37,345 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:27:37,346 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:27:37,349 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:27:37,351 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73622107, jitterRate=0.09705488383769989}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:27:37,351 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:27:37,353 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733344057323Writing region info on filesystem at 1733344057323Initializing all the Stores at 1733344057324 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733344057324Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733344057325 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344057325Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733344057325Cleaning up temporary data from old regions at 1733344057345 (+20 ms)Running coprocessor post-open hooks at 1733344057351 (+6 ms)Region opened successfully at 1733344057353 (+2 ms) 2024-12-04T20:27:37,361 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733344057262 2024-12-04T20:27:37,374 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:27:37,375 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:27:37,377 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:27:37,379 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,45443,1733344054291, state=OPEN 2024-12-04T20:27:37,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:27:37,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:27:37,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:27:37,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:27:37,383 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:37,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:27:37,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,45443,1733344054291 in 286 msec 2024-12-04T20:27:37,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:27:37,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 498 msec 2024-12-04T20:27:37,412 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:27:37,412 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:27:37,435 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:37,436 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:37,460 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:37,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34811, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:37,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5560 sec 2024-12-04T20:27:37,485 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733344057485, completionTime=-1 2024-12-04T20:27:37,488 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:27:37,489 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:27:37,520 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:27:37,520 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733344117520 2024-12-04T20:27:37,521 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733344177521 2024-12-04T20:27:37,521 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-12-04T20:27:37,523 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43507,1733344053664-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:37,523 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43507,1733344053664-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:37,524 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43507,1733344053664-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:37,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:43507, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:37,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:37,526 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:37,532 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:27:37,552 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.115sec 2024-12-04T20:27:37,553 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:27:37,554 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:27:37,555 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:27:37,555 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:27:37,555 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:27:37,556 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43507,1733344053664-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:27:37,556 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43507,1733344053664-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:27:37,564 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:27:37,565 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:27:37,565 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43507,1733344053664-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:27:37,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:37,605 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T20:27:37,605 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T20:27:37,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:37,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:37,617 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc9c1f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:37,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:37,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:37,628 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:37,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:37,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,43507,1733344053664 2024-12-04T20:27:37,668 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=4741 2024-12-04T20:27:37,676 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:27:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,43507,1733344053664 2024-12-04T20:27:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@219792a1 2024-12-04T20:27:37,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:27:37,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:27:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:27:37,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:27:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T20:27:37,737 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:27:37,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-04T20:27:37,740 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:37,743 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:27:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:27:37,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741835_1011 (size=1061) 2024-12-04T20:27:37,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:27:37,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741836_1012 (size=53) 2024-12-04T20:27:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:27:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:27:38,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:27:38,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e4d991821bf5842b6ec138d6f2a184fc, disabling compactions & flushes 2024-12-04T20:27:38,175 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. after waiting 0 ms 2024-12-04T20:27:38,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,175 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e4d991821bf5842b6ec138d6f2a184fc: Waiting for close lock at 1733344058175Disabling compacts and flushes for region at 1733344058175Disabling writes for close at 1733344058175Writing region close event to WAL at 1733344058175Closed at 1733344058175 2024-12-04T20:27:38,178 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:27:38,182 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733344058178"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733344058178"}]},"ts":"1733344058178"} 2024-12-04T20:27:38,186 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:27:38,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:27:38,190 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344058188"}]},"ts":"1733344058188"} 2024-12-04T20:27:38,194 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T20:27:38,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, ASSIGN}] 2024-12-04T20:27:38,198 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, ASSIGN 2024-12-04T20:27:38,200 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, ASSIGN; state=OFFLINE, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=false 2024-12-04T20:27:38,351 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e4d991821bf5842b6ec138d6f2a184fc, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:27:38,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, ASSIGN because future has completed 2024-12-04T20:27:38,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:27:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:27:38,517 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,517 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:27:38,518 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,518 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:27:38,518 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,518 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,520 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,523 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:27:38,524 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d991821bf5842b6ec138d6f2a184fc columnFamilyName A 2024-12-04T20:27:38,524 DEBUG [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:38,525 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.HStore(327): Store=e4d991821bf5842b6ec138d6f2a184fc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:27:38,525 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,527 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:27:38,527 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d991821bf5842b6ec138d6f2a184fc columnFamilyName B 2024-12-04T20:27:38,528 DEBUG [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:38,528 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.HStore(327): Store=e4d991821bf5842b6ec138d6f2a184fc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:27:38,528 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,530 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:27:38,530 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d991821bf5842b6ec138d6f2a184fc columnFamilyName C 2024-12-04T20:27:38,531 DEBUG [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:27:38,531 INFO [StoreOpener-e4d991821bf5842b6ec138d6f2a184fc-1 {}] regionserver.HStore(327): Store=e4d991821bf5842b6ec138d6f2a184fc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:27:38,532 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,532 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,534 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,535 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,537 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,537 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,539 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:27:38,541 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,545 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:27:38,546 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e4d991821bf5842b6ec138d6f2a184fc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65896085, jitterRate=-0.018071815371513367}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:27:38,546 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:38,547 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e4d991821bf5842b6ec138d6f2a184fc: Running coprocessor pre-open hook at 1733344058518Writing region info on filesystem at 1733344058518Initializing all the Stores at 1733344058520 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344058520Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344058520Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344058520Cleaning up temporary data from old regions at 1733344058538 (+18 ms)Running coprocessor post-open hooks at 1733344058546 (+8 ms)Region opened successfully at 1733344058547 (+1 ms) 2024-12-04T20:27:38,550 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., pid=6, masterSystemTime=1733344058510 2024-12-04T20:27:38,554 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,554 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:38,555 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e4d991821bf5842b6ec138d6f2a184fc, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:27:38,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:27:38,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T20:27:38,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 in 205 msec 2024-12-04T20:27:38,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T20:27:38,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, ASSIGN in 370 msec 2024-12-04T20:27:38,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:27:38,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344058572"}]},"ts":"1733344058572"} 2024-12-04T20:27:38,576 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T20:27:38,578 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:27:38,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 853 msec 2024-12-04T20:27:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:27:38,884 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:38,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,893 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f6a966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d4aab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:38,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:38,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:38,911 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:27:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,43507,1733344053664 2024-12-04T20:27:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@558238d3 2024-12-04T20:27:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:27:38,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:27:38,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,929 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,943 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,948 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,955 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,963 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee18a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,966 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256affd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,970 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b782e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,972 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b466e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,975 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1b5c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c0a0fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89bfc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,984 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e548029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb5cb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:38,994 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:27:38,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:27:38,998 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:27:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:27:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:27:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b0f1b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:27:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:27:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:39,001 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:27:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66945757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:27:39,009 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,009 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,010 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,011 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,012 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,013 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,019 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,024 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,038 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:27:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:27:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:27:39,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:27:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:27:39,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-04T20:27:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:27:39,068 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:39,071 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:39,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:39,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:27:39,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:39,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:39,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:39,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:39,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:39,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:27:39,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/88a6d8882e6c46ca9920b2fee543a323 is 50, key is test_row_0/A:col10/1733344059090/Put/seqid=0 2024-12-04T20:27:39,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:27:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344119221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344059193, max timestamp=1733344059231Num uniques 859; 2024-12-04T20:27:39,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:39,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:39,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344059193, max timestamp=1733344059227Num uniques 859; 2024-12-04T20:27:39,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344059193, max timestamp=1733344059231Num uniques 859; 2024-12-04T20:27:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:39,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741837_1013 (size=11997) 2024-12-04T20:27:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/88a6d8882e6c46ca9920b2fee543a323 2024-12-04T20:27:39,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:27:39,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344119348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344119354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344119352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344119389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:27:39,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/b992e8ca0c3440e68d638dadf6c0a9e0 is 50, key is test_row_0/B:col10/1733344059090/Put/seqid=0 2024-12-04T20:27:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344119462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344119495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344119502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344119503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344119514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741838_1014 (size=9657) 2024-12-04T20:27:39,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/b992e8ca0c3440e68d638dadf6c0a9e0 2024-12-04T20:27:39,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/a95fdab46c84417093f66653bc31c057 is 50, key is test_row_0/C:col10/1733344059090/Put/seqid=0 2024-12-04T20:27:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741839_1015 (size=9657) 2024-12-04T20:27:39,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:27:39,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344119683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:27:39,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344119713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344119713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344119722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344119723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:39,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:39,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:27:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:27:39,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:39,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:39,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:40,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344120004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/a95fdab46c84417093f66653bc31c057 2024-12-04T20:27:40,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344120021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344120022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/88a6d8882e6c46ca9920b2fee543a323 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/88a6d8882e6c46ca9920b2fee543a323 2024-12-04T20:27:40,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344120042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344120043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/88a6d8882e6c46ca9920b2fee543a323, entries=150, sequenceid=11, filesize=11.7 K 2024-12-04T20:27:40,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/b992e8ca0c3440e68d638dadf6c0a9e0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b992e8ca0c3440e68d638dadf6c0a9e0 2024-12-04T20:27:40,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b992e8ca0c3440e68d638dadf6c0a9e0, entries=100, sequenceid=11, filesize=9.4 K 2024-12-04T20:27:40,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/a95fdab46c84417093f66653bc31c057 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/a95fdab46c84417093f66653bc31c057 2024-12-04T20:27:40,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/a95fdab46c84417093f66653bc31c057, entries=100, sequenceid=11, filesize=9.4 K 2024-12-04T20:27:40,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:27:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:40,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:40,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for e4d991821bf5842b6ec138d6f2a184fc in 1021ms, sequenceid=11, compaction requested=false 2024-12-04T20:27:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T20:27:40,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:27:40,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:27:40,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:40,292 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-04T20:27:40,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:40,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/44daa267c1aa41b0ae9f90d5d9ff934e is 50, key is test_row_0/A:col10/1733344059381/Put/seqid=0 2024-12-04T20:27:40,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741840_1016 (size=12001) 2024-12-04T20:27:40,400 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/44daa267c1aa41b0ae9f90d5d9ff934e 2024-12-04T20:27:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/a93b35cd6c5d414696a31f42eec91f9f is 50, key is test_row_0/B:col10/1733344059381/Put/seqid=0 2024-12-04T20:27:40,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741841_1017 (size=12001) 2024-12-04T20:27:40,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:40,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344120556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344120561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344120567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344120567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344120572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344120672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344120682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344120683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344120683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344120683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,885 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/a93b35cd6c5d414696a31f42eec91f9f 2024-12-04T20:27:40,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344120892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344120903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344120904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344120904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344120904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:40,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/c1dc31eff59946b9a2e71ee17fe8c7f6 is 50, key is test_row_0/C:col10/1733344059381/Put/seqid=0 2024-12-04T20:27:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:40,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741842_1018 (size=12001) 2024-12-04T20:27:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:27:41,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344121212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344121222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344121222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344121232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344121232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,365 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/c1dc31eff59946b9a2e71ee17fe8c7f6 2024-12-04T20:27:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/44daa267c1aa41b0ae9f90d5d9ff934e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/44daa267c1aa41b0ae9f90d5d9ff934e 2024-12-04T20:27:41,401 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/44daa267c1aa41b0ae9f90d5d9ff934e, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T20:27:41,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/a93b35cd6c5d414696a31f42eec91f9f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a93b35cd6c5d414696a31f42eec91f9f 2024-12-04T20:27:41,418 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a93b35cd6c5d414696a31f42eec91f9f, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T20:27:41,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/c1dc31eff59946b9a2e71ee17fe8c7f6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/c1dc31eff59946b9a2e71ee17fe8c7f6 2024-12-04T20:27:41,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/c1dc31eff59946b9a2e71ee17fe8c7f6, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T20:27:41,438 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=60.38 KB/61830 for e4d991821bf5842b6ec138d6f2a184fc in 1146ms, sequenceid=48, compaction requested=false 2024-12-04T20:27:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T20:27:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T20:27:41,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T20:27:41,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-12-04T20:27:41,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3880 sec 2024-12-04T20:27:41,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:27:41,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:41,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:41,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:41,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:41,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:41,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:41,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6121de962bd042eda6a934484be589bc is 50, key is test_row_0/A:col10/1733344060556/Put/seqid=0 2024-12-04T20:27:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741843_1019 (size=14341) 2024-12-04T20:27:41,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6121de962bd042eda6a934484be589bc 2024-12-04T20:27:41,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/48944c6fab23441db6a5369f1d352143 is 50, key is test_row_0/B:col10/1733344060556/Put/seqid=0 2024-12-04T20:27:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344121819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344121827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344121827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344121820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344121821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741844_1020 (size=12001) 2024-12-04T20:27:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344121942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344121942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344121942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344121952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:41,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344121962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:42,134 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T20:27:42,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344122151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344122152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344122162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344122162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344122172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:42,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/48944c6fab23441db6a5369f1d352143 2024-12-04T20:27:42,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/213bf6faa5a94657bc5ff0afed0fbc62 is 50, key is test_row_0/C:col10/1733344060556/Put/seqid=0 2024-12-04T20:27:42,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741845_1021 (size=12001) 2024-12-04T20:27:42,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/213bf6faa5a94657bc5ff0afed0fbc62 2024-12-04T20:27:42,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6121de962bd042eda6a934484be589bc as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6121de962bd042eda6a934484be589bc 2024-12-04T20:27:42,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6121de962bd042eda6a934484be589bc, entries=200, sequenceid=61, filesize=14.0 K 2024-12-04T20:27:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/48944c6fab23441db6a5369f1d352143 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/48944c6fab23441db6a5369f1d352143 2024-12-04T20:27:42,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/48944c6fab23441db6a5369f1d352143, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T20:27:42,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/213bf6faa5a94657bc5ff0afed0fbc62 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/213bf6faa5a94657bc5ff0afed0fbc62 2024-12-04T20:27:42,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/213bf6faa5a94657bc5ff0afed0fbc62, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T20:27:42,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e4d991821bf5842b6ec138d6f2a184fc in 702ms, sequenceid=61, compaction requested=true 2024-12-04T20:27:42,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:42,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:42,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:42,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:42,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:42,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:42,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:42,457 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:42,461 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:42,461 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:42,463 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:42,463 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:42,464 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:42,464 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:42,464 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:42,464 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/88a6d8882e6c46ca9920b2fee543a323, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/44daa267c1aa41b0ae9f90d5d9ff934e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6121de962bd042eda6a934484be589bc] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=37.4 K 2024-12-04T20:27:42,464 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b992e8ca0c3440e68d638dadf6c0a9e0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a93b35cd6c5d414696a31f42eec91f9f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/48944c6fab23441db6a5369f1d352143] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=32.9 K 2024-12-04T20:27:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:42,466 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b992e8ca0c3440e68d638dadf6c0a9e0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733344059083 2024-12-04T20:27:42,466 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a6d8882e6c46ca9920b2fee543a323, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733344059064 2024-12-04T20:27:42,468 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44daa267c1aa41b0ae9f90d5d9ff934e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733344059321 2024-12-04T20:27:42,468 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a93b35cd6c5d414696a31f42eec91f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733344059321 2024-12-04T20:27:42,469 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 48944c6fab23441db6a5369f1d352143, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344060556 2024-12-04T20:27:42,469 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6121de962bd042eda6a934484be589bc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344060556 2024-12-04T20:27:42,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-04T20:27:42,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344061796, max timestamp=1733344062467Num uniques 859; 2024-12-04T20:27:42,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:42,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344061796, max timestamp=1733344062467Num uniques 859; 2024-12-04T20:27:42,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:42,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344061796, max timestamp=1733344062467Num uniques 859; 2024-12-04T20:27:42,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:42,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:42,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/92536861da054c89bcdd3ecf601e756c is 50, key is test_row_0/A:col10/1733344062467/Put/seqid=0 2024-12-04T20:27:42,567 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#9 average throughput is 0.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:42,568 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#10 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:42,569 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/8e712a839d5741e18cbdd77f7e4f2bb5 is 50, key is test_row_0/A:col10/1733344060556/Put/seqid=0 2024-12-04T20:27:42,570 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/174c40588db743aa891eaf583c6288bf is 50, key is test_row_0/B:col10/1733344060556/Put/seqid=0 2024-12-04T20:27:42,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741846_1022 (size=12001) 2024-12-04T20:27:42,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/92536861da054c89bcdd3ecf601e756c 2024-12-04T20:27:42,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741848_1024 (size=12104) 2024-12-04T20:27:42,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344062571, max timestamp=1733344062598Num uniques 859; 2024-12-04T20:27:42,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:42,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344062571, max timestamp=1733344062598Num uniques 859; 2024-12-04T20:27:42,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:42,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/ffcc266af1c54fa796242eeb97855149 is 50, key is test_row_0/B:col10/1733344062467/Put/seqid=0 2024-12-04T20:27:42,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344062571, max timestamp=1733344062594Num uniques 859; 2024-12-04T20:27:42,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:42,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741847_1023 (size=12104) 2024-12-04T20:27:42,667 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/174c40588db743aa891eaf583c6288bf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/174c40588db743aa891eaf583c6288bf 2024-12-04T20:27:42,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741849_1025 (size=12001) 2024-12-04T20:27:42,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/ffcc266af1c54fa796242eeb97855149 2024-12-04T20:27:42,710 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into 174c40588db743aa891eaf583c6288bf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:42,710 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:42,713 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=13, startTime=1733344062452; duration=0sec 2024-12-04T20:27:42,714 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:42,714 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:42,714 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:42,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f499868ac0be4af1bcd58524e795de49 is 50, key is test_row_0/C:col10/1733344062467/Put/seqid=0 2024-12-04T20:27:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344062679, max timestamp=1733344062703Num uniques 1018; 2024-12-04T20:27:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344062679, max timestamp=1733344062703Num uniques 1018; 2024-12-04T20:27:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T20:27:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T20:27:42,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344062683, max timestamp=1733344062703Num uniques 1018; 2024-12-04T20:27:42,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-04T20:27:42,745 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:42,745 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:42,745 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:42,746 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/a95fdab46c84417093f66653bc31c057, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/c1dc31eff59946b9a2e71ee17fe8c7f6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/213bf6faa5a94657bc5ff0afed0fbc62] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=32.9 K 2024-12-04T20:27:42,747 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a95fdab46c84417093f66653bc31c057, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733344059083 2024-12-04T20:27:42,748 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c1dc31eff59946b9a2e71ee17fe8c7f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733344059321 2024-12-04T20:27:42,753 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 213bf6faa5a94657bc5ff0afed0fbc62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344060556 2024-12-04T20:27:42,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741850_1026 (size=12001) 2024-12-04T20:27:42,793 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#14 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:42,794 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/6b2130f68397411ea5373581d4668fd6 is 50, key is test_row_0/C:col10/1733344060556/Put/seqid=0 2024-12-04T20:27:42,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741851_1027 (size=12104) 2024-12-04T20:27:42,832 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:27:42,832 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/6b2130f68397411ea5373581d4668fd6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6b2130f68397411ea5373581d4668fd6 2024-12-04T20:27:42,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733344062764, max timestamp=1733344062833Num uniques 1077; 2024-12-04T20:27:42,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733344062764, max timestamp=1733344062833Num uniques 1009; 2024-12-04T20:27:42,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T20:27:42,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46222, new segment data size=12682, suffix heap size=132904,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-12-04T20:27:42,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733344062764, max timestamp=1733344062833Num uniques 1077; 2024-12-04T20:27:42,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T20:27:42,862 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into 6b2130f68397411ea5373581d4668fd6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:42,862 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:42,862 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=13, startTime=1733344062455; duration=0sec 2024-12-04T20:27:42,862 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:42,863 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:42,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733344062891, max timestamp=1733344062917Num uniques 1136; 2024-12-04T20:27:42,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-12-04T20:27:42,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733344062891, max timestamp=1733344062917Num uniques 1136; 2024-12-04T20:27:42,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-04T20:27:42,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733344062891, max timestamp=1733344062917Num uniques 1136; 2024-12-04T20:27:42,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-04T20:27:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733344062957, max timestamp=1733344062983Num uniques 1145; 2024-12-04T20:27:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733344062957, max timestamp=1733344062983Num uniques 1045; 2024-12-04T20:27:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-12-04T20:27:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-12-04T20:27:42,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733344062971, max timestamp=1733344062983Num uniques 1045; 2024-12-04T20:27:42,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-04T20:27:43,021 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/8e712a839d5741e18cbdd77f7e4f2bb5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8e712a839d5741e18cbdd77f7e4f2bb5 2024-12-04T20:27:43,046 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into 8e712a839d5741e18cbdd77f7e4f2bb5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:43,046 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:43,046 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=13, startTime=1733344062434; duration=0sec 2024-12-04T20:27:43,046 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:43,046 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:43,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733344063036, max timestamp=1733344063070Num uniques 1204; 2024-12-04T20:27:43,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=13920, suffix heap size=153184,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=304 2024-12-04T20:27:43,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733344063041, max timestamp=1733344063073Num uniques 1054; 2024-12-04T20:27:43,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-04T20:27:43,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733344063036, max timestamp=1733344063070Num uniques 1204; 2024-12-04T20:27:43,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=13920, suffix heap size=153184,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=304 2024-12-04T20:27:43,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733344063113, max timestamp=1733344063140Num uniques 1163; 2024-12-04T20:27:43,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-12-04T20:27:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733344063113, max timestamp=1733344063140Num uniques 1013; 2024-12-04T20:27:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733344063113, max timestamp=1733344063140Num uniques 1163; 2024-12-04T20:27:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-12-04T20:27:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-04T20:27:43,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f499868ac0be4af1bcd58524e795de49 2024-12-04T20:27:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/92536861da054c89bcdd3ecf601e756c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/92536861da054c89bcdd3ecf601e756c 2024-12-04T20:27:43,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/92536861da054c89bcdd3ecf601e756c, entries=150, sequenceid=82, filesize=11.7 K 2024-12-04T20:27:43,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/ffcc266af1c54fa796242eeb97855149 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ffcc266af1c54fa796242eeb97855149 2024-12-04T20:27:43,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ffcc266af1c54fa796242eeb97855149, entries=150, sequenceid=82, filesize=11.7 K 2024-12-04T20:27:43,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f499868ac0be4af1bcd58524e795de49 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f499868ac0be4af1bcd58524e795de49 2024-12-04T20:27:43,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f499868ac0be4af1bcd58524e795de49, entries=150, sequenceid=82, filesize=11.7 K 2024-12-04T20:27:43,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.44 KB/27075, heapSize ~65.27 KB/66840, currentSize=134.18 KB/137400 for e4d991821bf5842b6ec138d6f2a184fc in 749ms, sequenceid=82, compaction requested=false 2024-12-04T20:27:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:27:43,222 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:43,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-04T20:27:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:27:43,226 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:43,227 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:43,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:43,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733344063203, max timestamp=1733344063229Num uniques 1022; 2024-12-04T20:27:43,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733344063203, max timestamp=1733344063229Num uniques 1022; 2024-12-04T20:27:43,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-04T20:27:43,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-04T20:27:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:43,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=71.56 KB heapSize=198.34 KB 2024-12-04T20:27:43,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733344063203, max timestamp=1733344063231Num uniques 1022; 2024-12-04T20:27:43,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-04T20:27:43,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:43,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:43,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:43,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:43,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:43,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:43,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0f582b1f33354a92a0417cf273ffc91b is 50, key is test_row_0/A:col10/1733344063203/Put/seqid=0 2024-12-04T20:27:43,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344063290, max timestamp=1733344063301Num uniques 859; 2024-12-04T20:27:43,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:43,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344063290, max timestamp=1733344063301Num uniques 859; 2024-12-04T20:27:43,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:43,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741852_1028 (size=14541) 2024-12-04T20:27:43,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344063290, max timestamp=1733344063301Num uniques 859; 2024-12-04T20:27:43,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:43,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0f582b1f33354a92a0417cf273ffc91b 2024-12-04T20:27:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:27:43,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/3d57b57411694ce18ddcfe2bd9724ee1 is 50, key is test_row_0/B:col10/1733344063203/Put/seqid=0 2024-12-04T20:27:43,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344123358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344123361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344063344, max timestamp=1733344063370Num uniques 1068; 2024-12-04T20:27:43,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:27:43,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344063344, max timestamp=1733344063370Num uniques 1068; 2024-12-04T20:27:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344123363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344063344, max timestamp=1733344063370Num uniques 1068; 2024-12-04T20:27:43,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:27:43,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:27:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741853_1029 (size=12151) 2024-12-04T20:27:43,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12674, totalHeapSize=29048, min timestamp=1733344063427, max timestamp=1733344063471Num uniques 1077; 2024-12-04T20:27:43,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49314, new segment data size=12674, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T20:27:43,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733344063427, max timestamp=1733344063471Num uniques 1077; 2024-12-04T20:27:43,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T20:27:43,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733344063427, max timestamp=1733344063471Num uniques 1077; 2024-12-04T20:27:43,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T20:27:43,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:43,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:27:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733344063538, max timestamp=1733344063567Num uniques 1136; 2024-12-04T20:27:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-04T20:27:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733344063538, max timestamp=1733344063567Num uniques 1136; 2024-12-04T20:27:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-04T20:27:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733344063538, max timestamp=1733344063567Num uniques 1136; 2024-12-04T20:27:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-04T20:27:43,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344123543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344123546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:43,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:43,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8910, totalHeapSize=20520, min timestamp=1733344063649, max timestamp=1733344063695Num uniques 1095; 2024-12-04T20:27:43,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344063649, max timestamp=1733344063695Num uniques 1095; 2024-12-04T20:27:43,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50137, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-12-04T20:27:43,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50130, new segment data size=8910, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-04T20:27:43,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344063649, max timestamp=1733344063695Num uniques 1095; 2024-12-04T20:27:43,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50130, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-12-04T20:27:43,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733344063772, max timestamp=1733344063794Num uniques 1054; 2024-12-04T20:27:43,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-04T20:27:43,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733344063772, max timestamp=1733344063794Num uniques 1009; 2024-12-04T20:27:43,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733344063772, max timestamp=1733344063794Num uniques 1009; 2024-12-04T20:27:43,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46223, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-12-04T20:27:43,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46230, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-12-04T20:27:43,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/3d57b57411694ce18ddcfe2bd9724ee1 2024-12-04T20:27:43,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/bd6c5019439746c7a9751663d97c5510 is 50, key is test_row_0/C:col10/1733344063203/Put/seqid=0 2024-12-04T20:27:43,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:27:43,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741854_1030 (size=12151) 2024-12-04T20:27:43,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733344063835, max timestamp=1733344063863Num uniques 1063; 2024-12-04T20:27:43,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-04T20:27:43,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733344063835, max timestamp=1733344063863Num uniques 1063; 2024-12-04T20:27:43,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-04T20:27:43,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344123850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733344063835, max timestamp=1733344063863Num uniques 1063; 2024-12-04T20:27:43,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-04T20:27:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/bd6c5019439746c7a9751663d97c5510 2024-12-04T20:27:43,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0f582b1f33354a92a0417cf273ffc91b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0f582b1f33354a92a0417cf273ffc91b 2024-12-04T20:27:43,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0f582b1f33354a92a0417cf273ffc91b, entries=200, sequenceid=228, filesize=14.2 K 2024-12-04T20:27:43,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/3d57b57411694ce18ddcfe2bd9724ee1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/3d57b57411694ce18ddcfe2bd9724ee1 2024-12-04T20:27:43,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/3d57b57411694ce18ddcfe2bd9724ee1, entries=150, sequenceid=228, filesize=11.9 K 2024-12-04T20:27:43,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/bd6c5019439746c7a9751663d97c5510 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/bd6c5019439746c7a9751663d97c5510 2024-12-04T20:27:43,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/bd6c5019439746c7a9751663d97c5510, entries=150, sequenceid=228, filesize=11.9 K 2024-12-04T20:27:43,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~104.81 KB/107328, currentSize=120.76 KB/123660 for e4d991821bf5842b6ec138d6f2a184fc in 700ms, sequenceid=228, compaction requested=true 2024-12-04T20:27:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:43,936 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:43,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:43,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T20:27:43,937 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:43,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=127.47 KB heapSize=364.69 KB 2024-12-04T20:27:43,939 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:43,939 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38646 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:43,939 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:43,939 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:43,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:43,939 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,939 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:43,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:43,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:43,939 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/174c40588db743aa891eaf583c6288bf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ffcc266af1c54fa796242eeb97855149, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/3d57b57411694ce18ddcfe2bd9724ee1] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=35.4 K 2024-12-04T20:27:43,939 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8e712a839d5741e18cbdd77f7e4f2bb5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/92536861da054c89bcdd3ecf601e756c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0f582b1f33354a92a0417cf273ffc91b] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=37.7 K 2024-12-04T20:27:43,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:43,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:43,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:43,941 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 174c40588db743aa891eaf583c6288bf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344060556 2024-12-04T20:27:43,941 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e712a839d5741e18cbdd77f7e4f2bb5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344060556 2024-12-04T20:27:43,942 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92536861da054c89bcdd3ecf601e756c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733344061820 2024-12-04T20:27:43,942 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting ffcc266af1c54fa796242eeb97855149, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733344061820 2024-12-04T20:27:43,943 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f582b1f33354a92a0417cf273ffc91b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733344063203 2024-12-04T20:27:43,944 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d57b57411694ce18ddcfe2bd9724ee1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733344063203 2024-12-04T20:27:43,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/1ae52f77548a497eb96309a66e315a0c is 50, key is test_row_0/A:col10/1733344063931/Put/seqid=0 2024-12-04T20:27:43,963 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:43,963 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:43,964 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f421c046723248a0b8742c4cd7783f15 is 50, key is test_row_0/B:col10/1733344063203/Put/seqid=0 2024-12-04T20:27:43,964 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/bef200616d3148a1a16af8c81a4b0922 is 50, key is test_row_0/A:col10/1733344063203/Put/seqid=0 2024-12-04T20:27:43,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344123972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344123973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344123978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:43,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344123982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344123985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741855_1031 (size=14741) 2024-12-04T20:27:44,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/1ae52f77548a497eb96309a66e315a0c 2024-12-04T20:27:44,027 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:27:44,027 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T20:27:44,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:27:44,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T20:27:44,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T20:27:44,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T20:27:44,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T20:27:44,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-04T20:27:44,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/23e8c0df73824163bc9cf56c9a863ad6 is 50, key is test_row_0/B:col10/1733344063931/Put/seqid=0 2024-12-04T20:27:44,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741856_1032 (size=12357) 2024-12-04T20:27:44,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741857_1033 (size=12357) 2024-12-04T20:27:44,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344124093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,101 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/bef200616d3148a1a16af8c81a4b0922 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/bef200616d3148a1a16af8c81a4b0922 2024-12-04T20:27:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741858_1034 (size=12301) 2024-12-04T20:27:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344124093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,115 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into bef200616d3148a1a16af8c81a4b0922(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:44,115 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:44,115 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=13, startTime=1733344063936; duration=0sec 2024-12-04T20:27:44,116 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:44,116 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:44,116 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:44,118 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:44,118 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:44,118 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,118 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6b2130f68397411ea5373581d4668fd6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f499868ac0be4af1bcd58524e795de49, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/bd6c5019439746c7a9751663d97c5510] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=35.4 K 2024-12-04T20:27:44,119 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2130f68397411ea5373581d4668fd6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344060556 2024-12-04T20:27:44,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344124112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344124112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,126 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f499868ac0be4af1bcd58524e795de49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733344061820 2024-12-04T20:27:44,128 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd6c5019439746c7a9751663d97c5510, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733344063203 2024-12-04T20:27:44,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,170 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:44,171 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/6cda30e7995741c2a43d70183f68a6e4 is 50, key is test_row_0/C:col10/1733344063203/Put/seqid=0 2024-12-04T20:27:44,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741859_1035 (size=12357) 2024-12-04T20:27:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344124212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,245 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/6cda30e7995741c2a43d70183f68a6e4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6cda30e7995741c2a43d70183f68a6e4 2024-12-04T20:27:44,257 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into 6cda30e7995741c2a43d70183f68a6e4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:44,257 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:44,257 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=13, startTime=1733344063937; duration=0sec 2024-12-04T20:27:44,258 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:44,258 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:44,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344124302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:44,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344124321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344124332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344124332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:27:44,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,480 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f421c046723248a0b8742c4cd7783f15 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f421c046723248a0b8742c4cd7783f15 2024-12-04T20:27:44,500 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into f421c046723248a0b8742c4cd7783f15(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:44,500 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:44,500 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=13, startTime=1733344063936; duration=0sec 2024-12-04T20:27:44,500 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:44,500 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:44,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/23e8c0df73824163bc9cf56c9a863ad6 2024-12-04T20:27:44,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/e45328019b434bc6bf655b717ec131c6 is 50, key is test_row_0/C:col10/1733344063931/Put/seqid=0 2024-12-04T20:27:44,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344124532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741860_1036 (size=12301) 2024-12-04T20:27:44,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344124622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344124632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344124652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344124652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:44,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:44,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:44,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:44,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/e45328019b434bc6bf655b717ec131c6 2024-12-04T20:27:44,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/1ae52f77548a497eb96309a66e315a0c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1ae52f77548a497eb96309a66e315a0c 2024-12-04T20:27:45,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1ae52f77548a497eb96309a66e315a0c, entries=200, sequenceid=367, filesize=14.4 K 2024-12-04T20:27:45,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/23e8c0df73824163bc9cf56c9a863ad6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/23e8c0df73824163bc9cf56c9a863ad6 2024-12-04T20:27:45,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/23e8c0df73824163bc9cf56c9a863ad6, entries=150, sequenceid=367, filesize=12.0 K 2024-12-04T20:27:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/e45328019b434bc6bf655b717ec131c6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e45328019b434bc6bf655b717ec131c6 2024-12-04T20:27:45,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e45328019b434bc6bf655b717ec131c6, entries=150, sequenceid=367, filesize=12.0 K 2024-12-04T20:27:45,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~364.64 KB/373392, currentSize=53.67 KB/54960 for e4d991821bf5842b6ec138d6f2a184fc in 1097ms, sequenceid=367, compaction requested=false 2024-12-04T20:27:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:45,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:27:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:45,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5a9307d1317e4122ab7f9a19e0fe9184 is 50, key is test_row_0/A:col10/1733344065054/Put/seqid=0 2024-12-04T20:27:45,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741861_1037 (size=14741) 2024-12-04T20:27:45,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5a9307d1317e4122ab7f9a19e0fe9184 2024-12-04T20:27:45,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/9db4387a1ea346e6820a28c19fca921d is 50, key is test_row_0/B:col10/1733344065054/Put/seqid=0 2024-12-04T20:27:45,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733344065162, max timestamp=1733344065178Num uniques 859; 2024-12-04T20:27:45,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733344065162, max timestamp=1733344065178Num uniques 859; 2024-12-04T20:27:45,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733344065162, max timestamp=1733344065178Num uniques 859; 2024-12-04T20:27:45,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T20:27:45,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T20:27:45,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T20:27:45,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741862_1038 (size=12301) 2024-12-04T20:27:45,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/9db4387a1ea346e6820a28c19fca921d 2024-12-04T20:27:45,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344125205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344125207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344125207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344125209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344125214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/e254218eaf794ab5b9376ecc9491d2e1 is 50, key is test_row_0/C:col10/1733344065054/Put/seqid=0 2024-12-04T20:27:45,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:45,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741863_1039 (size=12301) 2024-12-04T20:27:45,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/e254218eaf794ab5b9376ecc9491d2e1 2024-12-04T20:27:45,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5a9307d1317e4122ab7f9a19e0fe9184 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5a9307d1317e4122ab7f9a19e0fe9184 2024-12-04T20:27:45,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5a9307d1317e4122ab7f9a19e0fe9184, entries=200, sequenceid=382, filesize=14.4 K 2024-12-04T20:27:45,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/9db4387a1ea346e6820a28c19fca921d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9db4387a1ea346e6820a28c19fca921d 2024-12-04T20:27:45,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9db4387a1ea346e6820a28c19fca921d, entries=150, sequenceid=382, filesize=12.0 K 2024-12-04T20:27:45,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/e254218eaf794ab5b9376ecc9491d2e1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e254218eaf794ab5b9376ecc9491d2e1 2024-12-04T20:27:45,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e254218eaf794ab5b9376ecc9491d2e1, entries=150, sequenceid=382, filesize=12.0 K 2024-12-04T20:27:45,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for e4d991821bf5842b6ec138d6f2a184fc in 263ms, sequenceid=382, compaction requested=true 2024-12-04T20:27:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:45,319 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:45,320 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:45,321 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:45,321 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:45,321 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:45,321 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:45,321 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,321 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,322 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/bef200616d3148a1a16af8c81a4b0922, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1ae52f77548a497eb96309a66e315a0c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5a9307d1317e4122ab7f9a19e0fe9184] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=40.9 K 2024-12-04T20:27:45,322 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f421c046723248a0b8742c4cd7783f15, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/23e8c0df73824163bc9cf56c9a863ad6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9db4387a1ea346e6820a28c19fca921d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.1 K 2024-12-04T20:27:45,324 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting bef200616d3148a1a16af8c81a4b0922, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733344063203 2024-12-04T20:27:45,324 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f421c046723248a0b8742c4cd7783f15, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733344063203 2024-12-04T20:27:45,325 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae52f77548a497eb96309a66e315a0c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733344063876 2024-12-04T20:27:45,326 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9307d1317e4122ab7f9a19e0fe9184, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733344063965 2024-12-04T20:27:45,326 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e8c0df73824163bc9cf56c9a863ad6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733344063876 2024-12-04T20:27:45,328 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db4387a1ea346e6820a28c19fca921d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733344063970 2024-12-04T20:27:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:45,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-12-04T20:27:45,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:45,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:45,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:45,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:45,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:45,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:45,360 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:45,361 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/99011fa8c0e24987ba62a27865046443 is 50, key is test_row_0/B:col10/1733344065054/Put/seqid=0 2024-12-04T20:27:45,368 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:45,369 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/ed909a66e53b4ac9ae58c45f8fb3e006 is 50, key is test_row_0/A:col10/1733344065054/Put/seqid=0 2024-12-04T20:27:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:27:45,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344125364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344125345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344125367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344125365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344125371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/b5be0033da504a4494e5188f40f5646f is 50, key is test_row_0/A:col10/1733344065337/Put/seqid=0 2024-12-04T20:27:45,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:45,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:45,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741864_1040 (size=12609) 2024-12-04T20:27:45,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741865_1041 (size=12609) 2024-12-04T20:27:45,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741866_1042 (size=14741) 2024-12-04T20:27:45,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/b5be0033da504a4494e5188f40f5646f 2024-12-04T20:27:45,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344125482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,493 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/ed909a66e53b4ac9ae58c45f8fb3e006 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/ed909a66e53b4ac9ae58c45f8fb3e006 2024-12-04T20:27:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344125482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344125492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344125493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344125493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/bb4c4260ea8b465ba236ad5a5e6277b3 is 50, key is test_row_0/B:col10/1733344065337/Put/seqid=0 2024-12-04T20:27:45,518 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into ed909a66e53b4ac9ae58c45f8fb3e006(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:45,518 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:45,518 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=13, startTime=1733344065319; duration=0sec 2024-12-04T20:27:45,519 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:45,519 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:45,519 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:45,524 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:45,525 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:45,525 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,525 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6cda30e7995741c2a43d70183f68a6e4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e45328019b434bc6bf655b717ec131c6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e254218eaf794ab5b9376ecc9491d2e1] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.1 K 2024-12-04T20:27:45,526 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cda30e7995741c2a43d70183f68a6e4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733344063203 2024-12-04T20:27:45,527 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45328019b434bc6bf655b717ec131c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733344063876 2024-12-04T20:27:45,528 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e254218eaf794ab5b9376ecc9491d2e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733344063970 2024-12-04T20:27:45,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741867_1043 (size=12301) 2024-12-04T20:27:45,565 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:45,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:45,566 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/7cc8dd59a6c04a94a539418372d782fe is 50, key is test_row_0/C:col10/1733344065054/Put/seqid=0 2024-12-04T20:27:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741868_1044 (size=12609) 2024-12-04T20:27:45,604 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/7cc8dd59a6c04a94a539418372d782fe as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/7cc8dd59a6c04a94a539418372d782fe 2024-12-04T20:27:45,625 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into 7cc8dd59a6c04a94a539418372d782fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:45,629 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:45,629 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=13, startTime=1733344065320; duration=0sec 2024-12-04T20:27:45,629 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:45,630 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:45,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344125702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344125702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344125703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344125703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344125712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:45,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:45,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:45,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,843 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/99011fa8c0e24987ba62a27865046443 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/99011fa8c0e24987ba62a27865046443 2024-12-04T20:27:45,857 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into 99011fa8c0e24987ba62a27865046443(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:45,857 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:45,857 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=13, startTime=1733344065319; duration=0sec 2024-12-04T20:27:45,858 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:45,858 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:45,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:45,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:45,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:45,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:45,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/bb4c4260ea8b465ba236ad5a5e6277b3 2024-12-04T20:27:45,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/eba9b7fa6ff04e8f8442c2f1a0d52511 is 50, key is test_row_0/C:col10/1733344065337/Put/seqid=0 2024-12-04T20:27:46,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741869_1045 (size=12301) 2024-12-04T20:27:46,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/eba9b7fa6ff04e8f8442c2f1a0d52511 2024-12-04T20:27:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344126013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344126013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344126013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344126022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344126022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/b5be0033da504a4494e5188f40f5646f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/b5be0033da504a4494e5188f40f5646f 2024-12-04T20:27:46,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:46,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:46,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:46,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/b5be0033da504a4494e5188f40f5646f, entries=200, sequenceid=418, filesize=14.4 K 2024-12-04T20:27:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/bb4c4260ea8b465ba236ad5a5e6277b3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/bb4c4260ea8b465ba236ad5a5e6277b3 2024-12-04T20:27:46,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/bb4c4260ea8b465ba236ad5a5e6277b3, entries=150, sequenceid=418, filesize=12.0 K 2024-12-04T20:27:46,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/eba9b7fa6ff04e8f8442c2f1a0d52511 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/eba9b7fa6ff04e8f8442c2f1a0d52511 2024-12-04T20:27:46,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/eba9b7fa6ff04e8f8442c2f1a0d52511, entries=150, sequenceid=418, filesize=12.0 K 2024-12-04T20:27:46,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for e4d991821bf5842b6ec138d6f2a184fc in 726ms, sequenceid=418, compaction requested=false 2024-12-04T20:27:46,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:46,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:27:46,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:46,193 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T20:27:46,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:46,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:46,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:46,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:46,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:46,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/2b661786ffee4a5da39b37a3ba470800 is 50, key is test_row_0/A:col10/1733344065361/Put/seqid=0 2024-12-04T20:27:46,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741870_1046 (size=12301) 2024-12-04T20:27:46,265 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/2b661786ffee4a5da39b37a3ba470800 2024-12-04T20:27:46,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/349952afd52d419f9b5bb8d66eb08b43 is 50, key is test_row_0/B:col10/1733344065361/Put/seqid=0 2024-12-04T20:27:46,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741871_1047 (size=12301) 2024-12-04T20:27:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:46,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:46,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344066535, max timestamp=1733344066560Num uniques 859; 2024-12-04T20:27:46,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344066535, max timestamp=1733344066560Num uniques 859; 2024-12-04T20:27:46,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:46,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:46,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344066535, max timestamp=1733344066560Num uniques 859; 2024-12-04T20:27:46,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:46,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344066589, max timestamp=1733344066625Num uniques 1068; 2024-12-04T20:27:46,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:27:46,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344066589, max timestamp=1733344066628Num uniques 1068; 2024-12-04T20:27:46,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:27:46,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344066589, max timestamp=1733344066625Num uniques 1068; 2024-12-04T20:27:46,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T20:27:46,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733344066648, max timestamp=1733344066674Num uniques 1077; 2024-12-04T20:27:46,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733344066648, max timestamp=1733344066674Num uniques 1077; 2024-12-04T20:27:46,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-04T20:27:46,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-04T20:27:46,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733344066648, max timestamp=1733344066674Num uniques 1027; 2024-12-04T20:27:46,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-04T20:27:46,699 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/349952afd52d419f9b5bb8d66eb08b43 2024-12-04T20:27:46,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/deb2b064ea264f669f316be78a93ed1d is 50, key is test_row_0/C:col10/1733344065361/Put/seqid=0 2024-12-04T20:27:46,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741872_1048 (size=12301) 2024-12-04T20:27:46,734 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/deb2b064ea264f669f316be78a93ed1d 2024-12-04T20:27:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/2b661786ffee4a5da39b37a3ba470800 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2b661786ffee4a5da39b37a3ba470800 2024-12-04T20:27:46,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2b661786ffee4a5da39b37a3ba470800, entries=150, sequenceid=429, filesize=12.0 K 2024-12-04T20:27:46,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/349952afd52d419f9b5bb8d66eb08b43 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/349952afd52d419f9b5bb8d66eb08b43 2024-12-04T20:27:46,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344126753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344126758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344126760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,779 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/349952afd52d419f9b5bb8d66eb08b43, entries=150, sequenceid=429, filesize=12.0 K 2024-12-04T20:27:46,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/deb2b064ea264f669f316be78a93ed1d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/deb2b064ea264f669f316be78a93ed1d 2024-12-04T20:27:46,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344126770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344126771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:46,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/deb2b064ea264f669f316be78a93ed1d, entries=150, sequenceid=429, filesize=12.0 K 2024-12-04T20:27:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:46,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for e4d991821bf5842b6ec138d6f2a184fc in 601ms, sequenceid=429, compaction requested=true 2024-12-04T20:27:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T20:27:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T20:27:46,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T20:27:46,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5690 sec 2024-12-04T20:27:46,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 3.5760 sec 2024-12-04T20:27:46,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733344066751, max timestamp=1733344066883Num uniques 1186; 2024-12-04T20:27:46,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733344066751, max timestamp=1733344066883Num uniques 1186; 2024-12-04T20:27:46,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-04T20:27:46,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-04T20:27:46,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733344066751, max timestamp=1733344066883Num uniques 1186; 2024-12-04T20:27:46,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-04T20:27:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:46,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=53.67 KB heapSize=143.39 KB 2024-12-04T20:27:46,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:46,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:46,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:46,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:46,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:46,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:46,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/9142230c5220412ba8579cbc92afbef2 is 50, key is test_row_0/A:col10/1733344066894/Put/seqid=0 2024-12-04T20:27:46,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344066932, max timestamp=1733344066957Num uniques 859; 2024-12-04T20:27:46,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344066932, max timestamp=1733344066957Num uniques 859; 2024-12-04T20:27:46,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:46,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:46,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344066932, max timestamp=1733344066957Num uniques 859; 2024-12-04T20:27:46,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:46,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741873_1049 (size=12301) 2024-12-04T20:27:46,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/9142230c5220412ba8579cbc92afbef2 2024-12-04T20:27:46,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/a4a154985ac24566bf59338a37b4e4d1 is 50, key is test_row_0/B:col10/1733344066894/Put/seqid=0 2024-12-04T20:27:47,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741874_1050 (size=12301) 2024-12-04T20:27:47,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/a4a154985ac24566bf59338a37b4e4d1 2024-12-04T20:27:47,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/cdc7266631bb4b6eb3738a12133bf307 is 50, key is test_row_0/C:col10/1733344066894/Put/seqid=0 2024-12-04T20:27:47,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741875_1051 (size=12301) 2024-12-04T20:27:47,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/cdc7266631bb4b6eb3738a12133bf307 2024-12-04T20:27:47,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/9142230c5220412ba8579cbc92afbef2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/9142230c5220412ba8579cbc92afbef2 2024-12-04T20:27:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344127074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344127077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/9142230c5220412ba8579cbc92afbef2, entries=150, sequenceid=505, filesize=12.0 K 2024-12-04T20:27:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344127079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344127087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/a4a154985ac24566bf59338a37b4e4d1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a4a154985ac24566bf59338a37b4e4d1 2024-12-04T20:27:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344127065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a4a154985ac24566bf59338a37b4e4d1, entries=150, sequenceid=505, filesize=12.0 K 2024-12-04T20:27:47,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/cdc7266631bb4b6eb3738a12133bf307 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cdc7266631bb4b6eb3738a12133bf307 2024-12-04T20:27:47,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cdc7266631bb4b6eb3738a12133bf307, entries=150, sequenceid=505, filesize=12.0 K 2024-12-04T20:27:47,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~143.34 KB/146784, currentSize=132.99 KB/136185 for e4d991821bf5842b6ec138d6f2a184fc in 260ms, sequenceid=505, compaction requested=true 2024-12-04T20:27:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:47,163 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:47,163 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:47,166 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:47,166 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:47,166 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,166 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/99011fa8c0e24987ba62a27865046443, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/bb4c4260ea8b465ba236ad5a5e6277b3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/349952afd52d419f9b5bb8d66eb08b43, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a4a154985ac24566bf59338a37b4e4d1] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=48.4 K 2024-12-04T20:27:47,167 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51952 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:47,167 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:47,167 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,167 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/ed909a66e53b4ac9ae58c45f8fb3e006, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/b5be0033da504a4494e5188f40f5646f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2b661786ffee4a5da39b37a3ba470800, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/9142230c5220412ba8579cbc92afbef2] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=50.7 K 2024-12-04T20:27:47,168 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 99011fa8c0e24987ba62a27865046443, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733344063970 2024-12-04T20:27:47,169 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed909a66e53b4ac9ae58c45f8fb3e006, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733344063970 2024-12-04T20:27:47,170 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting bb4c4260ea8b465ba236ad5a5e6277b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733344065330 2024-12-04T20:27:47,171 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5be0033da504a4494e5188f40f5646f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733344065330 2024-12-04T20:27:47,171 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 349952afd52d419f9b5bb8d66eb08b43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733344065361 2024-12-04T20:27:47,172 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b661786ffee4a5da39b37a3ba470800, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733344065361 2024-12-04T20:27:47,173 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a154985ac24566bf59338a37b4e4d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1733344066894 2024-12-04T20:27:47,173 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9142230c5220412ba8579cbc92afbef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1733344066894 2024-12-04T20:27:47,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344067069, max timestamp=1733344067204Num uniques 1009; 2024-12-04T20:27:47,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344067069, max timestamp=1733344067204Num uniques 1009; 2024-12-04T20:27:47,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-04T20:27:47,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-04T20:27:47,207 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:47,208 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/abb05c32516a4e96895ee7b26fe259b2 is 50, key is test_row_0/B:col10/1733344066894/Put/seqid=0 2024-12-04T20:27:47,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344067069, max timestamp=1733344067204Num uniques 1009; 2024-12-04T20:27:47,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-04T20:27:47,219 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:47,220 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/d9fd230e17624d5fb37609cf328c6ffd is 50, key is test_row_0/A:col10/1733344066894/Put/seqid=0 2024-12-04T20:27:47,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741876_1052 (size=12745) 2024-12-04T20:27:47,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741877_1053 (size=12745) 2024-12-04T20:27:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:47,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-04T20:27:47,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:47,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:47,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:47,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:47,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:47,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:47,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/3624d67e9dc947958e3bb50bbd83270e is 50, key is test_row_0/A:col10/1733344067251/Put/seqid=0 2024-12-04T20:27:47,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741878_1054 (size=14741) 2024-12-04T20:27:47,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344067290, max timestamp=1733344067305Num uniques 859; 2024-12-04T20:27:47,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:47,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344067290, max timestamp=1733344067305Num uniques 859; 2024-12-04T20:27:47,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:47,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344067290, max timestamp=1733344067305Num uniques 859; 2024-12-04T20:27:47,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:47,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344127337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344127344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344127347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344127347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344127350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:27:47,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:47,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-04T20:27:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:27:47,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:47,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:47,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:47,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344127461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344127461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344127461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344127462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344127472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:27:47,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:47,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:47,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,636 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/abb05c32516a4e96895ee7b26fe259b2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/abb05c32516a4e96895ee7b26fe259b2 2024-12-04T20:27:47,646 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/d9fd230e17624d5fb37609cf328c6ffd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/d9fd230e17624d5fb37609cf328c6ffd 2024-12-04T20:27:47,663 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into abb05c32516a4e96895ee7b26fe259b2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:47,663 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:47,663 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=12, startTime=1733344067163; duration=0sec 2024-12-04T20:27:47,663 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:47,663 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:47,663 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:47,664 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into d9fd230e17624d5fb37609cf328c6ffd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:47,664 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:47,664 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=12, startTime=1733344067163; duration=0sec 2024-12-04T20:27:47,664 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:47,664 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:47,670 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:47,670 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:47,670 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,670 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/7cc8dd59a6c04a94a539418372d782fe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/eba9b7fa6ff04e8f8442c2f1a0d52511, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/deb2b064ea264f669f316be78a93ed1d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cdc7266631bb4b6eb3738a12133bf307] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=48.4 K 2024-12-04T20:27:47,672 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cc8dd59a6c04a94a539418372d782fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733344063970 2024-12-04T20:27:47,673 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting eba9b7fa6ff04e8f8442c2f1a0d52511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733344065330 2024-12-04T20:27:47,673 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting deb2b064ea264f669f316be78a93ed1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733344065361 2024-12-04T20:27:47,674 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting cdc7266631bb4b6eb3738a12133bf307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1733344066894 2024-12-04T20:27:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344127671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344127671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344127672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344127672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344127682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:47,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/3624d67e9dc947958e3bb50bbd83270e 2024-12-04T20:27:47,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:47,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:27:47,715 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:47,716 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/d76f564a70b04372b4437db98a9a33f5 is 50, key is test_row_0/C:col10/1733344066894/Put/seqid=0 2024-12-04T20:27:47,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f3e9e4754fc64a29ae34a829b580aaff is 50, key is test_row_0/B:col10/1733344067251/Put/seqid=0 2024-12-04T20:27:47,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741879_1055 (size=12745) 2024-12-04T20:27:47,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741880_1056 (size=12301) 2024-12-04T20:27:47,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f3e9e4754fc64a29ae34a829b580aaff 2024-12-04T20:27:47,801 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/d76f564a70b04372b4437db98a9a33f5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d76f564a70b04372b4437db98a9a33f5 2024-12-04T20:27:47,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/9a602bdaebcc4af7a700144a79070e22 is 50, key is test_row_0/C:col10/1733344067251/Put/seqid=0 2024-12-04T20:27:47,811 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into d76f564a70b04372b4437db98a9a33f5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:47,811 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:47,811 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=12, startTime=1733344067163; duration=0sec 2024-12-04T20:27:47,811 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:47,811 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:47,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741881_1057 (size=12301) 2024-12-04T20:27:47,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/9a602bdaebcc4af7a700144a79070e22 2024-12-04T20:27:47,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/3624d67e9dc947958e3bb50bbd83270e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/3624d67e9dc947958e3bb50bbd83270e 2024-12-04T20:27:47,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/3624d67e9dc947958e3bb50bbd83270e, entries=200, sequenceid=547, filesize=14.4 K 2024-12-04T20:27:47,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f3e9e4754fc64a29ae34a829b580aaff as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f3e9e4754fc64a29ae34a829b580aaff 2024-12-04T20:27:47,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f3e9e4754fc64a29ae34a829b580aaff, entries=150, sequenceid=547, filesize=12.0 K 2024-12-04T20:27:47,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/9a602bdaebcc4af7a700144a79070e22 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/9a602bdaebcc4af7a700144a79070e22 2024-12-04T20:27:47,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:47,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:47,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:47,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:47,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/9a602bdaebcc4af7a700144a79070e22, entries=150, sequenceid=547, filesize=12.0 K 2024-12-04T20:27:47,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=133.39 KB/136590 for e4d991821bf5842b6ec138d6f2a184fc in 619ms, sequenceid=547, compaction requested=false 2024-12-04T20:27:47,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:47,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=140.10 KB heapSize=403.78 KB 2024-12-04T20:27:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:27:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:27:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:27:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:48,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344127999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/da63ccddad054713af4377b99cbe464e is 50, key is test_row_0/A:col10/1733344067342/Put/seqid=0 2024-12-04T20:27:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344128003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344128003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344128006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:48,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:48,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344128007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:27:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741882_1058 (size=14741) 2024-12-04T20:27:48,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344128112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344128121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344128121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344128122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344128122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:48,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344128321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:48,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:48,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344128335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344128335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344128335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344128335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/da63ccddad054713af4377b99cbe464e 2024-12-04T20:27:48,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:48,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:48,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2abc7bcb51c547f483969b257c14b781 is 50, key is test_row_0/B:col10/1733344067342/Put/seqid=0 2024-12-04T20:27:48,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741883_1059 (size=12301) 2024-12-04T20:27:48,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2abc7bcb51c547f483969b257c14b781 2024-12-04T20:27:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:27:48,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/19c023100cea4bb3b068ad85b9af2b02 is 50, key is test_row_0/C:col10/1733344067342/Put/seqid=0 2024-12-04T20:27:48,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741884_1060 (size=12301) 2024-12-04T20:27:48,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344128632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:48,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344128643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344128644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344128652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344128644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:48,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:48,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:48,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:48,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:48,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/19c023100cea4bb3b068ad85b9af2b02 2024-12-04T20:27:48,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/da63ccddad054713af4377b99cbe464e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/da63ccddad054713af4377b99cbe464e 2024-12-04T20:27:48,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/da63ccddad054713af4377b99cbe464e, entries=200, sequenceid=588, filesize=14.4 K 2024-12-04T20:27:48,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2abc7bcb51c547f483969b257c14b781 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2abc7bcb51c547f483969b257c14b781 2024-12-04T20:27:49,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2abc7bcb51c547f483969b257c14b781, entries=150, sequenceid=588, filesize=12.0 K 2024-12-04T20:27:49,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/19c023100cea4bb3b068ad85b9af2b02 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/19c023100cea4bb3b068ad85b9af2b02 2024-12-04T20:27:49,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/19c023100cea4bb3b068ad85b9af2b02, entries=150, sequenceid=588, filesize=12.0 K 2024-12-04T20:27:49,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.10 KB/143460, heapSize ~403.73 KB/413424, currentSize=40.25 KB/41220 for e4d991821bf5842b6ec138d6f2a184fc in 1033ms, sequenceid=588, compaction requested=true 2024-12-04T20:27:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:49,021 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:49,021 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:49,022 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42227 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:49,022 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:49,022 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:49,022 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:49,022 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:49,023 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:49,023 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/d9fd230e17624d5fb37609cf328c6ffd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/3624d67e9dc947958e3bb50bbd83270e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/da63ccddad054713af4377b99cbe464e] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=41.2 K 2024-12-04T20:27:49,023 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/abb05c32516a4e96895ee7b26fe259b2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f3e9e4754fc64a29ae34a829b580aaff, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2abc7bcb51c547f483969b257c14b781] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.5 K 2024-12-04T20:27:49,023 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9fd230e17624d5fb37609cf328c6ffd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1733344066894 2024-12-04T20:27:49,023 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting abb05c32516a4e96895ee7b26fe259b2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1733344066894 2024-12-04T20:27:49,024 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3624d67e9dc947958e3bb50bbd83270e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733344067069 2024-12-04T20:27:49,024 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f3e9e4754fc64a29ae34a829b580aaff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733344067214 2024-12-04T20:27:49,024 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting da63ccddad054713af4377b99cbe464e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733344067342 2024-12-04T20:27:49,024 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2abc7bcb51c547f483969b257c14b781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733344067342 2024-12-04T20:27:49,048 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:49,049 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e86a220f58ef476985198c6f8535b672 is 50, key is test_row_0/A:col10/1733344067342/Put/seqid=0 2024-12-04T20:27:49,051 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:49,051 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/55c438a8ac544448a7be3e652a6f937c is 50, key is test_row_0/B:col10/1733344067342/Put/seqid=0 2024-12-04T20:27:49,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741885_1061 (size=12847) 2024-12-04T20:27:49,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741886_1062 (size=12847) 2024-12-04T20:27:49,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:27:49,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:49,103 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T20:27:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:49,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e448e6c192e54b12ba484d4713411170 is 50, key is test_row_0/A:col10/1733344067999/Put/seqid=0 2024-12-04T20:27:49,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741887_1063 (size=12301) 2024-12-04T20:27:49,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:49,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344069179, max timestamp=1733344069192Num uniques 859; 2024-12-04T20:27:49,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:49,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344069179, max timestamp=1733344069192Num uniques 859; 2024-12-04T20:27:49,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:49,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344069179, max timestamp=1733344069192Num uniques 859; 2024-12-04T20:27:49,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344129266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344129279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344129281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344129283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344129283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344129392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344129392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344129401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344129402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344129402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,479 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e86a220f58ef476985198c6f8535b672 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e86a220f58ef476985198c6f8535b672 2024-12-04T20:27:49,492 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/55c438a8ac544448a7be3e652a6f937c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/55c438a8ac544448a7be3e652a6f937c 2024-12-04T20:27:49,496 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into e86a220f58ef476985198c6f8535b672(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:49,496 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:49,496 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=13, startTime=1733344069021; duration=0sec 2024-12-04T20:27:49,496 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:49,496 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:49,496 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:49,498 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:49,498 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:49,498 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:49,499 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d76f564a70b04372b4437db98a9a33f5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/9a602bdaebcc4af7a700144a79070e22, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/19c023100cea4bb3b068ad85b9af2b02] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.5 K 2024-12-04T20:27:49,499 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d76f564a70b04372b4437db98a9a33f5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1733344066894 2024-12-04T20:27:49,504 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a602bdaebcc4af7a700144a79070e22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733344067214 2024-12-04T20:27:49,505 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c023100cea4bb3b068ad85b9af2b02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733344067342 2024-12-04T20:27:49,505 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into 55c438a8ac544448a7be3e652a6f937c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:49,505 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:49,505 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=13, startTime=1733344069021; duration=0sec 2024-12-04T20:27:49,507 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:49,507 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:49,516 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:49,517 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/71eecdf71c0640f6a7c6e89e68afd86d is 50, key is test_row_0/C:col10/1733344067342/Put/seqid=0 2024-12-04T20:27:49,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741888_1064 (size=12847) 2024-12-04T20:27:49,528 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e448e6c192e54b12ba484d4713411170 2024-12-04T20:27:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/73475dd4aa024b0ca29be02fd344348d is 50, key is test_row_0/B:col10/1733344067999/Put/seqid=0 2024-12-04T20:27:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:27:49,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741889_1065 (size=12301) 2024-12-04T20:27:49,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344129602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344129602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344129611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344129611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344129611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344129911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344129911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344129922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344129923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344129923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:49,938 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/71eecdf71c0640f6a7c6e89e68afd86d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/71eecdf71c0640f6a7c6e89e68afd86d 2024-12-04T20:27:49,952 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into 71eecdf71c0640f6a7c6e89e68afd86d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:49,953 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/73475dd4aa024b0ca29be02fd344348d 2024-12-04T20:27:49,953 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:49,953 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=13, startTime=1733344069021; duration=0sec 2024-12-04T20:27:49,953 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:49,953 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/b319ac8a0009466b8270684259773137 is 50, key is test_row_0/C:col10/1733344067999/Put/seqid=0 2024-12-04T20:27:49,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741890_1066 (size=12301) 2024-12-04T20:27:50,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/b319ac8a0009466b8270684259773137 2024-12-04T20:27:50,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e448e6c192e54b12ba484d4713411170 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e448e6c192e54b12ba484d4713411170 2024-12-04T20:27:50,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e448e6c192e54b12ba484d4713411170, entries=150, sequenceid=597, filesize=12.0 K 2024-12-04T20:27:50,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/73475dd4aa024b0ca29be02fd344348d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73475dd4aa024b0ca29be02fd344348d 2024-12-04T20:27:50,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73475dd4aa024b0ca29be02fd344348d, entries=150, sequenceid=597, filesize=12.0 K 2024-12-04T20:27:50,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/b319ac8a0009466b8270684259773137 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/b319ac8a0009466b8270684259773137 2024-12-04T20:27:50,047 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/b319ac8a0009466b8270684259773137, entries=150, sequenceid=597, filesize=12.0 K 2024-12-04T20:27:50,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for e4d991821bf5842b6ec138d6f2a184fc in 945ms, sequenceid=597, compaction requested=false 2024-12-04T20:27:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:50,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:50,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T20:27:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T20:27:50,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T20:27:50,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6530 sec 2024-12-04T20:27:50,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.6620 sec 2024-12-04T20:27:50,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:50,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344069247, max timestamp=1733344070423Num uniques 1068; 2024-12-04T20:27:50,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344069247, max timestamp=1733344070423Num uniques 1068; 2024-12-04T20:27:50,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T20:27:50,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T20:27:50,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=67.09 KB heapSize=183.88 KB 2024-12-04T20:27:50,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344069247, max timestamp=1733344070423Num uniques 1068; 2024-12-04T20:27:50,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T20:27:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:50,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0bb6719895b348349896dc41653d1757 is 50, key is test_row_0/A:col10/1733344070425/Put/seqid=0 2024-12-04T20:27:50,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733344070436, max timestamp=1733344070453Num uniques 859; 2024-12-04T20:27:50,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733344070436, max timestamp=1733344070453Num uniques 859; 2024-12-04T20:27:50,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-04T20:27:50,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-04T20:27:50,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733344070436, max timestamp=1733344070453Num uniques 859; 2024-12-04T20:27:50,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-04T20:27:50,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741891_1067 (size=12301) 2024-12-04T20:27:50,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0bb6719895b348349896dc41653d1757 2024-12-04T20:27:50,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344130506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344130508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344130510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344130513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344130513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2e4af94205c74ceb9bbee176b1859087 is 50, key is test_row_0/B:col10/1733344070425/Put/seqid=0 2024-12-04T20:27:50,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741892_1068 (size=12301) 2024-12-04T20:27:50,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2e4af94205c74ceb9bbee176b1859087 2024-12-04T20:27:50,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/d44428ec51084934ac96d72619426b3a is 50, key is test_row_0/C:col10/1733344070425/Put/seqid=0 2024-12-04T20:27:50,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741893_1069 (size=12301) 2024-12-04T20:27:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344130622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344130622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344130631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344130631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344130631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344130831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344130833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344130842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344130842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344130852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:50,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/d44428ec51084934ac96d72619426b3a 2024-12-04T20:27:50,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0bb6719895b348349896dc41653d1757 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0bb6719895b348349896dc41653d1757 2024-12-04T20:27:50,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0bb6719895b348349896dc41653d1757, entries=150, sequenceid=639, filesize=12.0 K 2024-12-04T20:27:50,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2e4af94205c74ceb9bbee176b1859087 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2e4af94205c74ceb9bbee176b1859087 2024-12-04T20:27:51,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2e4af94205c74ceb9bbee176b1859087, entries=150, sequenceid=639, filesize=12.0 K 2024-12-04T20:27:51,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/d44428ec51084934ac96d72619426b3a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d44428ec51084934ac96d72619426b3a 2024-12-04T20:27:51,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d44428ec51084934ac96d72619426b3a, entries=150, sequenceid=639, filesize=12.0 K 2024-12-04T20:27:51,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=174.43 KB/178620 for e4d991821bf5842b6ec138d6f2a184fc in 585ms, sequenceid=639, compaction requested=true 2024-12-04T20:27:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:51,011 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:51,011 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:51,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:51,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:51,016 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:51,016 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:51,016 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:51,016 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:51,016 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:51,016 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e86a220f58ef476985198c6f8535b672, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e448e6c192e54b12ba484d4713411170, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0bb6719895b348349896dc41653d1757] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.6 K 2024-12-04T20:27:51,017 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e86a220f58ef476985198c6f8535b672, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733344067342 2024-12-04T20:27:51,017 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:51,017 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e448e6c192e54b12ba484d4713411170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733344067999 2024-12-04T20:27:51,017 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/55c438a8ac544448a7be3e652a6f937c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73475dd4aa024b0ca29be02fd344348d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2e4af94205c74ceb9bbee176b1859087] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.6 K 2024-12-04T20:27:51,018 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bb6719895b348349896dc41653d1757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733344069279 2024-12-04T20:27:51,021 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 55c438a8ac544448a7be3e652a6f937c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733344067342 2024-12-04T20:27:51,022 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 73475dd4aa024b0ca29be02fd344348d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733344067999 2024-12-04T20:27:51,023 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e4af94205c74ceb9bbee176b1859087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733344069279 2024-12-04T20:27:51,047 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:51,048 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/38515607bd53431092b099ba0d819b92 is 50, key is test_row_0/A:col10/1733344070425/Put/seqid=0 2024-12-04T20:27:51,059 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:51,059 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/7e331e64429b41e4bae5002420da3c8f is 50, key is test_row_0/B:col10/1733344070425/Put/seqid=0 2024-12-04T20:27:51,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741894_1070 (size=12949) 2024-12-04T20:27:51,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741895_1071 (size=12949) 2024-12-04T20:27:51,124 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/7e331e64429b41e4bae5002420da3c8f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/7e331e64429b41e4bae5002420da3c8f 2024-12-04T20:27:51,140 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into 7e331e64429b41e4bae5002420da3c8f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:51,140 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:51,140 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=13, startTime=1733344071011; duration=0sec 2024-12-04T20:27:51,140 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:51,140 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:51,140 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:51,142 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:51,142 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:51,143 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:51,143 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/71eecdf71c0640f6a7c6e89e68afd86d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/b319ac8a0009466b8270684259773137, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d44428ec51084934ac96d72619426b3a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.6 K 2024-12-04T20:27:51,144 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 71eecdf71c0640f6a7c6e89e68afd86d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733344067342 2024-12-04T20:27:51,145 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b319ac8a0009466b8270684259773137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733344067999 2024-12-04T20:27:51,145 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d44428ec51084934ac96d72619426b3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733344069279 2024-12-04T20:27:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:51,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344070506, max timestamp=1733344071144Num uniques 1368; 2024-12-04T20:27:51,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344070506, max timestamp=1733344071144Num uniques 1368; 2024-12-04T20:27:51,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344070506, max timestamp=1733344071144Num uniques 1368; 2024-12-04T20:27:51,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=62648, new segment data size=9978, suffix heap size=170240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1368, new segment cells count=218 2024-12-04T20:27:51,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=62648, new segment data size=9978, suffix heap size=170240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1368, new segment cells count=218 2024-12-04T20:27:51,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=62648, new segment data size=9978, suffix heap size=170240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1368, new segment cells count=218 2024-12-04T20:27:51,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=33.54 KB heapSize=80.63 KB 2024-12-04T20:27:51,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:51,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:51,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:51,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:51,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:51,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:51,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e8d94c554e4a4f33a8da3932a077de0b is 50, key is test_row_0/A:col10/1733344070508/Put/seqid=0 2024-12-04T20:27:51,179 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:51,179 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/77f7ce872fb54e4fbc490e83d123bd4d is 50, key is test_row_0/C:col10/1733344070425/Put/seqid=0 2024-12-04T20:27:51,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1733344071170, max timestamp=1733344071188Num uniques 859; 2024-12-04T20:27:51,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T20:27:51,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1733344071170, max timestamp=1733344071188Num uniques 859; 2024-12-04T20:27:51,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T20:27:51,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344071178, max timestamp=1733344071188Num uniques 859; 2024-12-04T20:27:51,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741896_1072 (size=17181) 2024-12-04T20:27:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344131218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344131221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344131222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344131225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344131227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741897_1073 (size=12949) 2024-12-04T20:27:51,253 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/77f7ce872fb54e4fbc490e83d123bd4d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/77f7ce872fb54e4fbc490e83d123bd4d 2024-12-04T20:27:51,266 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into 77f7ce872fb54e4fbc490e83d123bd4d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:51,267 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:51,267 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=13, startTime=1733344071012; duration=0sec 2024-12-04T20:27:51,267 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:51,267 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:51,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344131331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344131331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344131342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344131342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344131342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,499 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/38515607bd53431092b099ba0d819b92 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/38515607bd53431092b099ba0d819b92 2024-12-04T20:27:51,514 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into 38515607bd53431092b099ba0d819b92(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:51,514 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:51,514 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=13, startTime=1733344071011; duration=0sec 2024-12-04T20:27:51,514 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:51,514 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:51,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344131541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344131542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344131553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344131553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344131553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:27:51,562 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:51,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-04T20:27:51,566 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T20:27:51,567 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:51,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:51,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e8d94c554e4a4f33a8da3932a077de0b 2024-12-04T20:27:51,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/8c5c8fd70fa5410ebd9ac925f124ee7c is 50, key is test_row_0/B:col10/1733344070508/Put/seqid=0 2024-12-04T20:27:51,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741898_1074 (size=12301) 2024-12-04T20:27:51,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/8c5c8fd70fa5410ebd9ac925f124ee7c 2024-12-04T20:27:51,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/4d32ace258b24bf38a639551883e5cfe is 50, key is test_row_0/C:col10/1733344070508/Put/seqid=0 2024-12-04T20:27:51,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741899_1075 (size=12301) 2024-12-04T20:27:51,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/4d32ace258b24bf38a639551883e5cfe 2024-12-04T20:27:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T20:27:51,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/e8d94c554e4a4f33a8da3932a077de0b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e8d94c554e4a4f33a8da3932a077de0b 2024-12-04T20:27:51,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e8d94c554e4a4f33a8da3932a077de0b, entries=250, sequenceid=678, filesize=16.8 K 2024-12-04T20:27:51,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/8c5c8fd70fa5410ebd9ac925f124ee7c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/8c5c8fd70fa5410ebd9ac925f124ee7c 2024-12-04T20:27:51,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/8c5c8fd70fa5410ebd9ac925f124ee7c, entries=150, sequenceid=678, filesize=12.0 K 2024-12-04T20:27:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/4d32ace258b24bf38a639551883e5cfe as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/4d32ace258b24bf38a639551883e5cfe 2024-12-04T20:27:51,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/4d32ace258b24bf38a639551883e5cfe, entries=150, sequenceid=678, filesize=12.0 K 2024-12-04T20:27:51,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=156.54 KB/160300 for e4d991821bf5842b6ec138d6f2a184fc in 566ms, sequenceid=678, compaction requested=false 2024-12-04T20:27:51,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:51,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T20:27:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:51,721 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=156.54 KB heapSize=435.77 KB 2024-12-04T20:27:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:51,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/638271dacf3347b19f1ef7cfb8727266 is 50, key is test_row_0/A:col10/1733344071225/Put/seqid=0 2024-12-04T20:27:51,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741900_1076 (size=12301) 2024-12-04T20:27:51,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/638271dacf3347b19f1ef7cfb8727266 2024-12-04T20:27:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/ad48be111cbd4b7eb24b08a3b990803f is 50, key is test_row_0/B:col10/1733344071225/Put/seqid=0 2024-12-04T20:27:51,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741901_1077 (size=12301) 2024-12-04T20:27:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:51,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344131859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344131862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344131862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344131862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344131862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T20:27:51,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344131971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344131971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344132181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344132181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T20:27:52,222 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/ad48be111cbd4b7eb24b08a3b990803f 2024-12-04T20:27:52,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/8329860e3f9543779d06069a1507ba2a is 50, key is test_row_0/C:col10/1733344071225/Put/seqid=0 2024-12-04T20:27:52,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741902_1078 (size=12301) 2024-12-04T20:27:52,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/8329860e3f9543779d06069a1507ba2a 2024-12-04T20:27:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/638271dacf3347b19f1ef7cfb8727266 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/638271dacf3347b19f1ef7cfb8727266 2024-12-04T20:27:52,304 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/638271dacf3347b19f1ef7cfb8727266, entries=150, sequenceid=717, filesize=12.0 K 2024-12-04T20:27:52,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/ad48be111cbd4b7eb24b08a3b990803f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ad48be111cbd4b7eb24b08a3b990803f 2024-12-04T20:27:52,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ad48be111cbd4b7eb24b08a3b990803f, entries=150, sequenceid=717, filesize=12.0 K 2024-12-04T20:27:52,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/8329860e3f9543779d06069a1507ba2a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/8329860e3f9543779d06069a1507ba2a 2024-12-04T20:27:52,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/8329860e3f9543779d06069a1507ba2a, entries=150, sequenceid=717, filesize=12.0 K 2024-12-04T20:27:52,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~156.54 KB/160300, heapSize ~435.72 KB/446176, currentSize=26.84 KB/27480 for e4d991821bf5842b6ec138d6f2a184fc in 619ms, sequenceid=717, compaction requested=true 2024-12-04T20:27:52,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:52,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:52,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T20:27:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T20:27:52,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T20:27:52,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 775 msec 2024-12-04T20:27:52,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 782 msec 2024-12-04T20:27:52,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:27:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:52,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/72fbd44c39e24131acbf21fba348ed68 is 50, key is test_row_0/A:col10/1733344072372/Put/seqid=0 2024-12-04T20:27:52,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741903_1079 (size=14741) 2024-12-04T20:27:52,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/72fbd44c39e24131acbf21fba348ed68 2024-12-04T20:27:52,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344072418, max timestamp=1733344072445Num uniques 859; 2024-12-04T20:27:52,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:52,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344072418, max timestamp=1733344072445Num uniques 859; 2024-12-04T20:27:52,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:52,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344072418, max timestamp=1733344072445Num uniques 859; 2024-12-04T20:27:52,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T20:27:52,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/692b6dc4cf9e4aa08bf37204363a1e6d is 50, key is test_row_0/B:col10/1733344072372/Put/seqid=0 2024-12-04T20:27:52,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741904_1080 (size=12301) 2024-12-04T20:27:52,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344132503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344132503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344132504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344132506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344132515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344132622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344132622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344132622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344132622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344132631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T20:27:52,702 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:52,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-04T20:27:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T20:27:52,705 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:52,706 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:52,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T20:27:52,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344132832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344132833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344132833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344132833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344132841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:52,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T20:27:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:52,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:52,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/692b6dc4cf9e4aa08bf37204363a1e6d 2024-12-04T20:27:52,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f1cf400146f2409b882f1fb315d87d70 is 50, key is test_row_0/C:col10/1733344072372/Put/seqid=0 2024-12-04T20:27:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741905_1081 (size=12301) 2024-12-04T20:27:52,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f1cf400146f2409b882f1fb315d87d70 2024-12-04T20:27:52,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/72fbd44c39e24131acbf21fba348ed68 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/72fbd44c39e24131acbf21fba348ed68 2024-12-04T20:27:52,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/72fbd44c39e24131acbf21fba348ed68, entries=200, sequenceid=727, filesize=14.4 K 2024-12-04T20:27:53,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/692b6dc4cf9e4aa08bf37204363a1e6d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/692b6dc4cf9e4aa08bf37204363a1e6d 2024-12-04T20:27:53,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/692b6dc4cf9e4aa08bf37204363a1e6d, entries=150, sequenceid=727, filesize=12.0 K 2024-12-04T20:27:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f1cf400146f2409b882f1fb315d87d70 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f1cf400146f2409b882f1fb315d87d70 2024-12-04T20:27:53,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f1cf400146f2409b882f1fb315d87d70, entries=150, sequenceid=727, filesize=12.0 K 2024-12-04T20:27:53,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for e4d991821bf5842b6ec138d6f2a184fc in 639ms, sequenceid=727, compaction requested=true 2024-12-04T20:27:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:53,017 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:53,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T20:27:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T20:27:53,017 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:53,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:53,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-04T20:27:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T20:27:53,023 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57172 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:53,023 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:53,024 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:53,024 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:53,029 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:53,029 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:53,030 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/38515607bd53431092b099ba0d819b92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e8d94c554e4a4f33a8da3932a077de0b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/638271dacf3347b19f1ef7cfb8727266, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/72fbd44c39e24131acbf21fba348ed68] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=55.8 K 2024-12-04T20:27:53,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/caa66c089d434998bdb88122c067a8d5 is 50, key is test_row_0/A:col10/1733344072501/Put/seqid=0 2024-12-04T20:27:53,034 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/7e331e64429b41e4bae5002420da3c8f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/8c5c8fd70fa5410ebd9ac925f124ee7c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ad48be111cbd4b7eb24b08a3b990803f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/692b6dc4cf9e4aa08bf37204363a1e6d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=48.7 K 2024-12-04T20:27:53,034 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e331e64429b41e4bae5002420da3c8f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733344069279 2024-12-04T20:27:53,036 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 38515607bd53431092b099ba0d819b92, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733344069279 2024-12-04T20:27:53,037 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c5c8fd70fa5410ebd9ac925f124ee7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733344070508 2024-12-04T20:27:53,037 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d94c554e4a4f33a8da3932a077de0b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733344070506 2024-12-04T20:27:53,039 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 638271dacf3347b19f1ef7cfb8727266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1733344071214 2024-12-04T20:27:53,039 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad48be111cbd4b7eb24b08a3b990803f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1733344071214 2024-12-04T20:27:53,040 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692b6dc4cf9e4aa08bf37204363a1e6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733344071855 2024-12-04T20:27:53,040 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 72fbd44c39e24131acbf21fba348ed68, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733344071855 2024-12-04T20:27:53,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741906_1082 (size=12301) 2024-12-04T20:27:53,087 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:53,088 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/60b707a58b8c44e8a2748ef0f7bc131f is 50, key is test_row_0/B:col10/1733344072372/Put/seqid=0 2024-12-04T20:27:53,095 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:53,096 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/42d0793463c542ddaa08e09d1a4cc21a is 50, key is test_row_0/A:col10/1733344072372/Put/seqid=0 2024-12-04T20:27:53,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741907_1083 (size=13085) 2024-12-04T20:27:53,144 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/60b707a58b8c44e8a2748ef0f7bc131f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60b707a58b8c44e8a2748ef0f7bc131f 2024-12-04T20:27:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:53,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:53,152 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into 60b707a58b8c44e8a2748ef0f7bc131f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:53,152 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:53,152 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=12, startTime=1733344073017; duration=0sec 2024-12-04T20:27:53,152 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:53,152 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:53,152 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:53,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741908_1084 (size=13085) 2024-12-04T20:27:53,154 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:53,155 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:53,155 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:53,155 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/77f7ce872fb54e4fbc490e83d123bd4d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/4d32ace258b24bf38a639551883e5cfe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/8329860e3f9543779d06069a1507ba2a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f1cf400146f2409b882f1fb315d87d70] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=48.7 K 2024-12-04T20:27:53,156 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f7ce872fb54e4fbc490e83d123bd4d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1733344069279 2024-12-04T20:27:53,157 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d32ace258b24bf38a639551883e5cfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733344070508 2024-12-04T20:27:53,157 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8329860e3f9543779d06069a1507ba2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1733344071214 2024-12-04T20:27:53,158 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1cf400146f2409b882f1fb315d87d70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733344071855 2024-12-04T20:27:53,165 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/42d0793463c542ddaa08e09d1a4cc21a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/42d0793463c542ddaa08e09d1a4cc21a 2024-12-04T20:27:53,174 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into 42d0793463c542ddaa08e09d1a4cc21a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:53,174 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:53,174 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=12, startTime=1733344073017; duration=0sec 2024-12-04T20:27:53,175 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:53,175 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:53,187 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:53,188 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/ad02adf8a5214628a1c61165e1c7708c is 50, key is test_row_0/C:col10/1733344072372/Put/seqid=0 2024-12-04T20:27:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344133175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344133178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344133178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344133181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344133187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741909_1085 (size=13085) 2024-12-04T20:27:53,211 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/ad02adf8a5214628a1c61165e1c7708c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ad02adf8a5214628a1c61165e1c7708c 2024-12-04T20:27:53,219 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into ad02adf8a5214628a1c61165e1c7708c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:53,219 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:53,219 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=12, startTime=1733344073017; duration=0sec 2024-12-04T20:27:53,219 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:53,219 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:53,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344133302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344133302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344133302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344133302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344133303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T20:27:53,488 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=764 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/caa66c089d434998bdb88122c067a8d5 2024-12-04T20:27:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/60774f7db88a417e9db704427c2cfe60 is 50, key is test_row_0/B:col10/1733344072501/Put/seqid=0 2024-12-04T20:27:53,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344133512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344133512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344133512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344133512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344133521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741910_1086 (size=12301) 2024-12-04T20:27:53,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=764 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/60774f7db88a417e9db704427c2cfe60 2024-12-04T20:27:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/2464a02f149a436bb19ac93d22cbb49a is 50, key is test_row_0/C:col10/1733344072501/Put/seqid=0 2024-12-04T20:27:53,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741911_1087 (size=12301) 2024-12-04T20:27:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344133822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344133822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344133822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344133831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344133831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T20:27:54,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=764 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/2464a02f149a436bb19ac93d22cbb49a 2024-12-04T20:27:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/caa66c089d434998bdb88122c067a8d5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/caa66c089d434998bdb88122c067a8d5 2024-12-04T20:27:54,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/caa66c089d434998bdb88122c067a8d5, entries=150, sequenceid=764, filesize=12.0 K 2024-12-04T20:27:54,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/60774f7db88a417e9db704427c2cfe60 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60774f7db88a417e9db704427c2cfe60 2024-12-04T20:27:54,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60774f7db88a417e9db704427c2cfe60, entries=150, sequenceid=764, filesize=12.0 K 2024-12-04T20:27:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/2464a02f149a436bb19ac93d22cbb49a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2464a02f149a436bb19ac93d22cbb49a 2024-12-04T20:27:54,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2464a02f149a436bb19ac93d22cbb49a, entries=150, sequenceid=764, filesize=12.0 K 2024-12-04T20:27:54,049 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for e4d991821bf5842b6ec138d6f2a184fc in 1032ms, sequenceid=764, compaction requested=false 2024-12-04T20:27:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-04T20:27:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-04T20:27:54,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T20:27:54,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3550 sec 2024-12-04T20:27:54,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.3680 sec 2024-12-04T20:27:54,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T20:27:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:54,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/8a1d35cec6154e799216ed0c7dcc0a7c is 50, key is test_row_0/A:col10/1733344073177/Put/seqid=0 2024-12-04T20:27:54,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344134373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344134386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344134386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344134387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344134388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741912_1088 (size=14741) 2024-12-04T20:27:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344134502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344134502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344134502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344134502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344134502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344134721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344134722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344134722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344134722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344134722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:54,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=779 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/8a1d35cec6154e799216ed0c7dcc0a7c 2024-12-04T20:27:54,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/c680fd5174a14699b36edd748cd13e9c is 50, key is test_row_0/B:col10/1733344073177/Put/seqid=0 2024-12-04T20:27:54,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741913_1089 (size=12301) 2024-12-04T20:27:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T20:27:54,853 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:54,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-04T20:27:54,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T20:27:54,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:54,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T20:27:55,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T20:27:55,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:55,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:55,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:55,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344135042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344135042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344135042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344135042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344135042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T20:27:55,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:55,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T20:27:55,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=779 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/c680fd5174a14699b36edd748cd13e9c 2024-12-04T20:27:55,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f7f45f0e1559463ea0dd25951a8d3345 is 50, key is test_row_0/C:col10/1733344073177/Put/seqid=0 2024-12-04T20:27:55,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741914_1090 (size=12301) 2024-12-04T20:27:55,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=779 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f7f45f0e1559463ea0dd25951a8d3345 2024-12-04T20:27:55,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/8a1d35cec6154e799216ed0c7dcc0a7c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8a1d35cec6154e799216ed0c7dcc0a7c 2024-12-04T20:27:55,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8a1d35cec6154e799216ed0c7dcc0a7c, entries=200, sequenceid=779, filesize=14.4 K 2024-12-04T20:27:55,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/c680fd5174a14699b36edd748cd13e9c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/c680fd5174a14699b36edd748cd13e9c 2024-12-04T20:27:55,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/c680fd5174a14699b36edd748cd13e9c, entries=150, sequenceid=779, filesize=12.0 K 2024-12-04T20:27:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/f7f45f0e1559463ea0dd25951a8d3345 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f7f45f0e1559463ea0dd25951a8d3345 2024-12-04T20:27:55,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f7f45f0e1559463ea0dd25951a8d3345, entries=150, sequenceid=779, filesize=12.0 K 2024-12-04T20:27:55,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e4d991821bf5842b6ec138d6f2a184fc in 966ms, sequenceid=779, compaction requested=true 2024-12-04T20:27:55,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:55,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:55,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:55,302 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T20:27:55,303 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:55,304 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:55,304 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:55,304 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,305 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60b707a58b8c44e8a2748ef0f7bc131f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60774f7db88a417e9db704427c2cfe60, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/c680fd5174a14699b36edd748cd13e9c] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.8 K 2024-12-04T20:27:55,305 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:55,305 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:55,305 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,305 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/42d0793463c542ddaa08e09d1a4cc21a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/caa66c089d434998bdb88122c067a8d5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8a1d35cec6154e799216ed0c7dcc0a7c] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=39.2 K 2024-12-04T20:27:55,306 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d0793463c542ddaa08e09d1a4cc21a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733344071855 2024-12-04T20:27:55,306 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b707a58b8c44e8a2748ef0f7bc131f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733344071855 2024-12-04T20:27:55,307 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa66c089d434998bdb88122c067a8d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=764, earliestPutTs=1733344072501 2024-12-04T20:27:55,307 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 60774f7db88a417e9db704427c2cfe60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=764, earliestPutTs=1733344072501 2024-12-04T20:27:55,307 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a1d35cec6154e799216ed0c7dcc0a7c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733344073177 2024-12-04T20:27:55,307 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c680fd5174a14699b36edd748cd13e9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733344073177 2024-12-04T20:27:55,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T20:27:55,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,322 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:55,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T20:27:55,322 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/81a9117d0cbf4849ab384ad7789d187e is 50, key is test_row_0/A:col10/1733344073177/Put/seqid=0 2024-12-04T20:27:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:55,334 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:55,335 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/73efbe5d6de14590b9c809376b2dccd5 is 50, key is test_row_0/B:col10/1733344073177/Put/seqid=0 2024-12-04T20:27:55,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741915_1091 (size=13187) 2024-12-04T20:27:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741916_1092 (size=13187) 2024-12-04T20:27:55,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/65e5d14619034478b56f3742413d15cf is 50, key is test_row_0/A:col10/1733344074364/Put/seqid=0 2024-12-04T20:27:55,350 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/81a9117d0cbf4849ab384ad7789d187e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/81a9117d0cbf4849ab384ad7789d187e 2024-12-04T20:27:55,361 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into 81a9117d0cbf4849ab384ad7789d187e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:55,361 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:55,361 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=13, startTime=1733344075302; duration=0sec 2024-12-04T20:27:55,361 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:55,361 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:55,361 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:27:55,364 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:27:55,364 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:55,364 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,365 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ad02adf8a5214628a1c61165e1c7708c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2464a02f149a436bb19ac93d22cbb49a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f7f45f0e1559463ea0dd25951a8d3345] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=36.8 K 2024-12-04T20:27:55,368 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad02adf8a5214628a1c61165e1c7708c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733344071855 2024-12-04T20:27:55,369 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2464a02f149a436bb19ac93d22cbb49a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=764, earliestPutTs=1733344072501 2024-12-04T20:27:55,370 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7f45f0e1559463ea0dd25951a8d3345, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733344073177 2024-12-04T20:27:55,393 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:55,394 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/28de17b30cb849c98de67503d23a5eb8 is 50, key is test_row_0/C:col10/1733344073177/Put/seqid=0 2024-12-04T20:27:55,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741917_1093 (size=12301) 2024-12-04T20:27:55,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/65e5d14619034478b56f3742413d15cf 2024-12-04T20:27:55,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/9925e8a44b9d4295a9ea3f18df68941e is 50, key is test_row_0/B:col10/1733344074364/Put/seqid=0 2024-12-04T20:27:55,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741918_1094 (size=13187) 2024-12-04T20:27:55,447 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/28de17b30cb849c98de67503d23a5eb8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/28de17b30cb849c98de67503d23a5eb8 2024-12-04T20:27:55,454 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into 28de17b30cb849c98de67503d23a5eb8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:55,454 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:55,454 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=13, startTime=1733344075303; duration=0sec 2024-12-04T20:27:55,454 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:55,454 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:55,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741919_1095 (size=12301) 2024-12-04T20:27:55,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/9925e8a44b9d4295a9ea3f18df68941e 2024-12-04T20:27:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T20:27:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/2ad7264ec1fc400faacff40b09b0e33f is 50, key is test_row_0/C:col10/1733344074364/Put/seqid=0 2024-12-04T20:27:55,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741920_1096 (size=12301) 2024-12-04T20:27:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:55,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344135558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344135562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344135563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344135565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344135571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344135671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344135685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344135685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344135686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344135686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,752 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/73efbe5d6de14590b9c809376b2dccd5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73efbe5d6de14590b9c809376b2dccd5 2024-12-04T20:27:55,761 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into 73efbe5d6de14590b9c809376b2dccd5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:55,762 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:55,762 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=13, startTime=1733344075302; duration=0sec 2024-12-04T20:27:55,762 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:55,762 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:55,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344135882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344135892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344135892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344135892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344135901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:55,936 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/2ad7264ec1fc400faacff40b09b0e33f 2024-12-04T20:27:55,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/65e5d14619034478b56f3742413d15cf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/65e5d14619034478b56f3742413d15cf 2024-12-04T20:27:55,952 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/65e5d14619034478b56f3742413d15cf, entries=150, sequenceid=799, filesize=12.0 K 2024-12-04T20:27:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/9925e8a44b9d4295a9ea3f18df68941e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9925e8a44b9d4295a9ea3f18df68941e 2024-12-04T20:27:55,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9925e8a44b9d4295a9ea3f18df68941e, entries=150, sequenceid=799, filesize=12.0 K 2024-12-04T20:27:55,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/2ad7264ec1fc400faacff40b09b0e33f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2ad7264ec1fc400faacff40b09b0e33f 2024-12-04T20:27:55,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2ad7264ec1fc400faacff40b09b0e33f, entries=150, sequenceid=799, filesize=12.0 K 2024-12-04T20:27:55,977 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e4d991821bf5842b6ec138d6f2a184fc in 655ms, sequenceid=799, compaction requested=false 2024-12-04T20:27:55,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:55,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:55,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-04T20:27:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-04T20:27:55,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T20:27:55,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1220 sec 2024-12-04T20:27:55,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1270 sec 2024-12-04T20:27:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T20:27:55,992 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:55,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-04T20:27:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T20:27:55,995 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:55,996 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:55,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T20:27:56,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T20:27:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:56,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:27:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0357d2dc36b4499ab471ec7c7cb6fdc9 is 50, key is test_row_0/A:col10/1733344075563/Put/seqid=0 2024-12-04T20:27:56,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741921_1097 (size=12301) 2024-12-04T20:27:56,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0357d2dc36b4499ab471ec7c7cb6fdc9 2024-12-04T20:27:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:56,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f41a47e6b95743bb87e848ce2da32233 is 50, key is test_row_0/B:col10/1733344075563/Put/seqid=0 2024-12-04T20:27:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344136237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344136239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344136240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344136241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344136241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741922_1098 (size=12301) 2024-12-04T20:27:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T20:27:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344136352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344136352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344136352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344136352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344136353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344136561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344136561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344136561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344136562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344136562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T20:27:56,655 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f41a47e6b95743bb87e848ce2da32233 2024-12-04T20:27:56,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/69897ac88c9f43369a3b289f540f63fd is 50, key is test_row_0/C:col10/1733344075563/Put/seqid=0 2024-12-04T20:27:56,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741923_1099 (size=12301) 2024-12-04T20:27:56,669 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/69897ac88c9f43369a3b289f540f63fd 2024-12-04T20:27:56,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/0357d2dc36b4499ab471ec7c7cb6fdc9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0357d2dc36b4499ab471ec7c7cb6fdc9 2024-12-04T20:27:56,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0357d2dc36b4499ab471ec7c7cb6fdc9, entries=150, sequenceid=815, filesize=12.0 K 2024-12-04T20:27:56,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/f41a47e6b95743bb87e848ce2da32233 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f41a47e6b95743bb87e848ce2da32233 2024-12-04T20:27:56,688 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f41a47e6b95743bb87e848ce2da32233, entries=150, sequenceid=815, filesize=12.0 K 2024-12-04T20:27:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/69897ac88c9f43369a3b289f540f63fd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/69897ac88c9f43369a3b289f540f63fd 2024-12-04T20:27:56,696 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/69897ac88c9f43369a3b289f540f63fd, entries=150, sequenceid=815, filesize=12.0 K 2024-12-04T20:27:56,697 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for e4d991821bf5842b6ec138d6f2a184fc in 547ms, sequenceid=815, compaction requested=true 2024-12-04T20:27:56,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-04T20:27:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-04T20:27:56,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-04T20:27:56,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 707 msec 2024-12-04T20:27:56,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 715 msec 2024-12-04T20:27:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:56,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T20:27:56,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344076237, max timestamp=1733344076874Num uniques 859; 2024-12-04T20:27:56,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:27:56,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:56,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:56,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:56,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:56,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:56,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:56,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/1cede52a9c4b4485b23f51c55bd680e0 is 50, key is test_row_0/A:col10/1733344076875/Put/seqid=0 2024-12-04T20:27:56,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344136900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344136904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344136906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344136906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344136906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:56,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741924_1100 (size=17181) 2024-12-04T20:27:57,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344137011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344137011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344137021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344137021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344137022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T20:27:57,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:27:57,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:27:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-04T20:27:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T20:27:57,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:27:57,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:27:57,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:27:57,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344137222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344137222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344137231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344137231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344137232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T20:27:57,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/1cede52a9c4b4485b23f51c55bd680e0 2024-12-04T20:27:57,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2ebfd866b05a4e7f941c20b0317253c8 is 50, key is test_row_0/B:col10/1733344076875/Put/seqid=0 2024-12-04T20:27:57,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741925_1101 (size=12301) 2024-12-04T20:27:57,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2ebfd866b05a4e7f941c20b0317253c8 2024-12-04T20:27:57,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/16f4d8b7ffc04483b87ecbcd00277dbb is 50, key is test_row_0/C:col10/1733344076875/Put/seqid=0 2024-12-04T20:27:57,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741926_1102 (size=12301) 2024-12-04T20:27:57,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/16f4d8b7ffc04483b87ecbcd00277dbb 2024-12-04T20:27:57,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:57,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/1cede52a9c4b4485b23f51c55bd680e0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1cede52a9c4b4485b23f51c55bd680e0 2024-12-04T20:27:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T20:27:57,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1cede52a9c4b4485b23f51c55bd680e0, entries=250, sequenceid=837, filesize=16.8 K 2024-12-04T20:27:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/2ebfd866b05a4e7f941c20b0317253c8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2ebfd866b05a4e7f941c20b0317253c8 2024-12-04T20:27:57,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2ebfd866b05a4e7f941c20b0317253c8, entries=150, sequenceid=837, filesize=12.0 K 2024-12-04T20:27:57,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/16f4d8b7ffc04483b87ecbcd00277dbb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/16f4d8b7ffc04483b87ecbcd00277dbb 2024-12-04T20:27:57,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/16f4d8b7ffc04483b87ecbcd00277dbb, entries=150, sequenceid=837, filesize=12.0 K 2024-12-04T20:27:57,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=87.22 KB/89310 for e4d991821bf5842b6ec138d6f2a184fc in 639ms, sequenceid=837, compaction requested=true 2024-12-04T20:27:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:57,515 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:57,516 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:57,516 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:27:57,517 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,517 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/81a9117d0cbf4849ab384ad7789d187e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/65e5d14619034478b56f3742413d15cf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0357d2dc36b4499ab471ec7c7cb6fdc9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1cede52a9c4b4485b23f51c55bd680e0] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=53.7 K 2024-12-04T20:27:57,517 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81a9117d0cbf4849ab384ad7789d187e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733344073177 2024-12-04T20:27:57,518 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e5d14619034478b56f3742413d15cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1733344074364 2024-12-04T20:27:57,518 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0357d2dc36b4499ab471ec7c7cb6fdc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1733344075562 2024-12-04T20:27:57,518 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cede52a9c4b4485b23f51c55bd680e0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1733344076237 2024-12-04T20:27:57,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:27:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:27:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:57,526 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:27:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:57,527 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:57,527 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:27:57,527 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,528 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73efbe5d6de14590b9c809376b2dccd5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9925e8a44b9d4295a9ea3f18df68941e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f41a47e6b95743bb87e848ce2da32233, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2ebfd866b05a4e7f941c20b0317253c8] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=48.9 K 2024-12-04T20:27:57,528 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 73efbe5d6de14590b9c809376b2dccd5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733344073177 2024-12-04T20:27:57,528 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9925e8a44b9d4295a9ea3f18df68941e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1733344074364 2024-12-04T20:27:57,529 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f41a47e6b95743bb87e848ce2da32233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1733344075562 2024-12-04T20:27:57,529 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ebfd866b05a4e7f941c20b0317253c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1733344076238 2024-12-04T20:27:57,538 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:57,539 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5852dbfc965547feb3fb37cbd4aa2d00 is 50, key is test_row_0/A:col10/1733344076875/Put/seqid=0 2024-12-04T20:27:57,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T20:27:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:57,549 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:57,551 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/e9e76fc9b3d54761b9e2b4566b9bfbcf is 50, key is test_row_0/B:col10/1733344076875/Put/seqid=0 2024-12-04T20:27:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:57,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6661dbf0f74e45b7bf68c821896e3513 is 50, key is test_row_0/A:col10/1733344077549/Put/seqid=0 2024-12-04T20:27:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741927_1103 (size=13323) 2024-12-04T20:27:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344137578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344137578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344137587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344137587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344137589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741928_1104 (size=13323) 2024-12-04T20:27:57,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741929_1105 (size=19621) 2024-12-04T20:27:57,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6661dbf0f74e45b7bf68c821896e3513 2024-12-04T20:27:57,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/fff7a25bb2644d53a66fa8ab229fb9a4 is 50, key is test_row_0/B:col10/1733344077549/Put/seqid=0 2024-12-04T20:27:57,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344137702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344137702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344137702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344137702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344137702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741930_1106 (size=12301) 2024-12-04T20:27:57,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:57,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T20:27:57,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:57,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344137911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344137922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344137922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344137923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344137923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:57,988 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5852dbfc965547feb3fb37cbd4aa2d00 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5852dbfc965547feb3fb37cbd4aa2d00 2024-12-04T20:27:58,001 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into 5852dbfc965547feb3fb37cbd4aa2d00(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:58,001 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:58,001 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=12, startTime=1733344077515; duration=0sec 2024-12-04T20:27:58,002 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:27:58,002 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:27:58,002 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:27:58,004 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:27:58,004 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:27:58,004 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,004 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/28de17b30cb849c98de67503d23a5eb8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2ad7264ec1fc400faacff40b09b0e33f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/69897ac88c9f43369a3b289f540f63fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/16f4d8b7ffc04483b87ecbcd00277dbb] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=48.9 K 2024-12-04T20:27:58,004 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28de17b30cb849c98de67503d23a5eb8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733344073177 2024-12-04T20:27:58,005 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad7264ec1fc400faacff40b09b0e33f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1733344074364 2024-12-04T20:27:58,005 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69897ac88c9f43369a3b289f540f63fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1733344075562 2024-12-04T20:27:58,006 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f4d8b7ffc04483b87ecbcd00277dbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1733344076238 2024-12-04T20:27:58,022 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:27:58,023 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/cb052562c89040c2b7630fef1380ab9d is 50, key is test_row_0/C:col10/1733344076875/Put/seqid=0 2024-12-04T20:27:58,024 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/e9e76fc9b3d54761b9e2b4566b9bfbcf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/e9e76fc9b3d54761b9e2b4566b9bfbcf 2024-12-04T20:27:58,031 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into e9e76fc9b3d54761b9e2b4566b9bfbcf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:58,031 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:58,032 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=12, startTime=1733344077526; duration=0sec 2024-12-04T20:27:58,032 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:58,032 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:27:58,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741931_1107 (size=13323) 2024-12-04T20:27:58,048 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/cb052562c89040c2b7630fef1380ab9d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cb052562c89040c2b7630fef1380ab9d 2024-12-04T20:27:58,054 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into cb052562c89040c2b7630fef1380ab9d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:27:58,054 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:58,054 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=12, startTime=1733344077526; duration=0sec 2024-12-04T20:27:58,054 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:27:58,054 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:27:58,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:58,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/fff7a25bb2644d53a66fa8ab229fb9a4 2024-12-04T20:27:58,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/aa54468220064b4a8e81eb1e76faef31 is 50, key is test_row_0/C:col10/1733344077549/Put/seqid=0 2024-12-04T20:27:58,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741932_1108 (size=12301) 2024-12-04T20:27:58,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344138221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344138232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344138232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344138232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344138232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T20:27:58,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:58,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:58,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:27:58,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/aa54468220064b4a8e81eb1e76faef31 2024-12-04T20:27:58,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6661dbf0f74e45b7bf68c821896e3513 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6661dbf0f74e45b7bf68c821896e3513 2024-12-04T20:27:58,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6661dbf0f74e45b7bf68c821896e3513, entries=300, sequenceid=856, filesize=19.2 K 2024-12-04T20:27:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/fff7a25bb2644d53a66fa8ab229fb9a4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/fff7a25bb2644d53a66fa8ab229fb9a4 2024-12-04T20:27:58,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/fff7a25bb2644d53a66fa8ab229fb9a4, entries=150, sequenceid=856, filesize=12.0 K 2024-12-04T20:27:58,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/aa54468220064b4a8e81eb1e76faef31 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/aa54468220064b4a8e81eb1e76faef31 2024-12-04T20:27:58,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/aa54468220064b4a8e81eb1e76faef31, entries=150, sequenceid=856, filesize=12.0 K 2024-12-04T20:27:58,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e4d991821bf5842b6ec138d6f2a184fc in 1069ms, sequenceid=856, compaction requested=false 2024-12-04T20:27:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:58,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T20:27:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:58,676 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:27:58,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:27:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65 is 50, key is test_row_0/A:col10/1733344077551/Put/seqid=0 2024-12-04T20:27:58,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741933_1109 (size=12301) 2024-12-04T20:27:58,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. as already flushing 2024-12-04T20:27:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344138753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344138754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344138755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344138756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344138761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344138871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344138872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344138872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344138872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344138872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,055 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-04T20:27:59,055 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-04T20:27:59,055 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,055 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,056 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-04T20:27:59,056 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,057 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-04T20:27:59,057 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,060 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,060 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,060 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,060 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,060 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,060 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,060 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,061 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344139082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344139082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344139082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344139082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344139082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,095 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65 2024-12-04T20:27:59,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/38ce3f8d82154b7f8c8c656c714ac8e1 is 50, key is test_row_0/B:col10/1733344077551/Put/seqid=0 2024-12-04T20:27:59,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741934_1110 (size=12301) 2024-12-04T20:27:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T20:27:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56868 deadline: 1733344139392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56846 deadline: 1733344139392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56862 deadline: 1733344139392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:27:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56840 deadline: 1733344139392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56822 deadline: 1733344139392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:27:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:27:59,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/38ce3f8d82154b7f8c8c656c714ac8e1 2024-12-04T20:27:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/0ee85ecf46dc464e84064a6ed363e801 is 50, key is test_row_0/C:col10/1733344077551/Put/seqid=0 2024-12-04T20:27:59,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741935_1111 (size=12301) 2024-12-04T20:27:59,524 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/0ee85ecf46dc464e84064a6ed363e801 2024-12-04T20:27:59,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65 2024-12-04T20:27:59,534 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65, entries=150, sequenceid=872, filesize=12.0 K 2024-12-04T20:27:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/38ce3f8d82154b7f8c8c656c714ac8e1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/38ce3f8d82154b7f8c8c656c714ac8e1 2024-12-04T20:27:59,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/38ce3f8d82154b7f8c8c656c714ac8e1, entries=150, sequenceid=872, filesize=12.0 K 2024-12-04T20:27:59,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/0ee85ecf46dc464e84064a6ed363e801 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/0ee85ecf46dc464e84064a6ed363e801 2024-12-04T20:27:59,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/0ee85ecf46dc464e84064a6ed363e801, entries=150, sequenceid=872, filesize=12.0 K 2024-12-04T20:27:59,546 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e4d991821bf5842b6ec138d6f2a184fc in 870ms, sequenceid=872, compaction requested=true 2024-12-04T20:27:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:27:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:27:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-04T20:27:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-04T20:27:59,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-04T20:27:59,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4100 sec 2024-12-04T20:27:59,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.4160 sec 2024-12-04T20:27:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:27:59,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4d991821bf5842b6ec138d6f2a184fc 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-04T20:27:59,904 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-04T20:27:59,904 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-04T20:27:59,904 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,904 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,904 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,904 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,904 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,904 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1733344078753, max timestamp=1733344079902Num uniques 859; 2024-12-04T20:27:59,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:59,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9561, totalHeapSize=21976, min timestamp=1733344078753, max timestamp=1733344079902Num uniques 859; 2024-12-04T20:27:59,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39331, new segment data size=9561, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:27:59,905 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-04T20:27:59,905 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,905 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,905 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,906 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-04T20:27:59,906 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-04T20:27:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=A 2024-12-04T20:27:59,906 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,906 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:27:59,906 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:59,906 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=B 2024-12-04T20:27:59,906 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,906 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:27:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e4d991821bf5842b6ec138d6f2a184fc, store=C 2024-12-04T20:27:59,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:27:59,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:27:59,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5cb55e240ad8464fba094364b0fd8612 is 50, key is test_row_0/A:col10/1733344079904/Put/seqid=0 2024-12-04T20:27:59,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741936_1112 (size=12301) 2024-12-04T20:28:00,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=896 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5cb55e240ad8464fba094364b0fd8612 2024-12-04T20:28:00,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/77029b6190c04f0fa979ace9c8796c0d is 50, key is test_row_0/B:col10/1733344079904/Put/seqid=0 2024-12-04T20:28:00,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741937_1113 (size=12301) 2024-12-04T20:28:00,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=896 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/77029b6190c04f0fa979ace9c8796c0d 2024-12-04T20:28:00,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/83873d089999406da66f7fa68af29e05 is 50, key is test_row_0/C:col10/1733344079904/Put/seqid=0 2024-12-04T20:28:00,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741938_1114 (size=12301) 2024-12-04T20:28:01,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=896 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/83873d089999406da66f7fa68af29e05 2024-12-04T20:28:01,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/5cb55e240ad8464fba094364b0fd8612 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5cb55e240ad8464fba094364b0fd8612 2024-12-04T20:28:01,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5cb55e240ad8464fba094364b0fd8612, entries=150, sequenceid=896, filesize=12.0 K 2024-12-04T20:28:01,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/77029b6190c04f0fa979ace9c8796c0d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/77029b6190c04f0fa979ace9c8796c0d 2024-12-04T20:28:01,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/77029b6190c04f0fa979ace9c8796c0d, entries=150, sequenceid=896, filesize=12.0 K 2024-12-04T20:28:01,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/83873d089999406da66f7fa68af29e05 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/83873d089999406da66f7fa68af29e05 2024-12-04T20:28:01,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/83873d089999406da66f7fa68af29e05, entries=150, sequenceid=896, filesize=12.0 K 2024-12-04T20:28:01,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~82.74 KB/84730, heapSize ~234.05 KB/239664, currentSize=0 B/0 for e4d991821bf5842b6ec138d6f2a184fc in 1260ms, sequenceid=896, compaction requested=true 2024-12-04T20:28:01,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:28:01,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:01,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:01,164 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:01,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:01,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:01,164 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:01,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4d991821bf5842b6ec138d6f2a184fc:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:01,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:01,165 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:01,165 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:01,165 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/B is initiating minor compaction (all files) 2024-12-04T20:28:01,165 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/A is initiating minor compaction (all files) 2024-12-04T20:28:01,165 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/B in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,165 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/A in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,165 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5852dbfc965547feb3fb37cbd4aa2d00, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6661dbf0f74e45b7bf68c821896e3513, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5cb55e240ad8464fba094364b0fd8612] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=56.2 K 2024-12-04T20:28:01,165 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/e9e76fc9b3d54761b9e2b4566b9bfbcf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/fff7a25bb2644d53a66fa8ab229fb9a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/38ce3f8d82154b7f8c8c656c714ac8e1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/77029b6190c04f0fa979ace9c8796c0d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=49.0 K 2024-12-04T20:28:01,166 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5852dbfc965547feb3fb37cbd4aa2d00, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1733344076238 2024-12-04T20:28:01,166 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e9e76fc9b3d54761b9e2b4566b9bfbcf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1733344076238 2024-12-04T20:28:01,166 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6661dbf0f74e45b7bf68c821896e3513, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1733344076898 2024-12-04T20:28:01,166 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting fff7a25bb2644d53a66fa8ab229fb9a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1733344076900 2024-12-04T20:28:01,166 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7541e9ef6b4dc6a6d7c8ef5f679f65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1733344077551 2024-12-04T20:28:01,166 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ce3f8d82154b7f8c8c656c714ac8e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1733344077551 2024-12-04T20:28:01,167 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb55e240ad8464fba094364b0fd8612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=896, earliestPutTs=1733344078754 2024-12-04T20:28:01,167 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 77029b6190c04f0fa979ace9c8796c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=896, earliestPutTs=1733344078754 2024-12-04T20:28:01,175 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:01,175 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/2dae9c2453b14e43958030e399fb321e is 50, key is test_row_0/A:col10/1733344079904/Put/seqid=0 2024-12-04T20:28:01,178 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#B#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:01,179 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/b5e3067f036c40428e2a49269f1914b2 is 50, key is test_row_0/B:col10/1733344079904/Put/seqid=0 2024-12-04T20:28:01,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741939_1115 (size=13459) 2024-12-04T20:28:01,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741940_1116 (size=13459) 2024-12-04T20:28:01,188 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/A/2dae9c2453b14e43958030e399fb321e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2dae9c2453b14e43958030e399fb321e 2024-12-04T20:28:01,193 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/A of e4d991821bf5842b6ec138d6f2a184fc into 2dae9c2453b14e43958030e399fb321e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:01,193 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:28:01,193 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/A, priority=12, startTime=1733344081164; duration=0sec 2024-12-04T20:28:01,194 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:01,194 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:A 2024-12-04T20:28:01,194 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:01,195 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:01,195 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): e4d991821bf5842b6ec138d6f2a184fc/C is initiating minor compaction (all files) 2024-12-04T20:28:01,195 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4d991821bf5842b6ec138d6f2a184fc/C in TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,195 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cb052562c89040c2b7630fef1380ab9d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/aa54468220064b4a8e81eb1e76faef31, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/0ee85ecf46dc464e84064a6ed363e801, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/83873d089999406da66f7fa68af29e05] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp, totalSize=49.0 K 2024-12-04T20:28:01,195 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb052562c89040c2b7630fef1380ab9d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1733344076238 2024-12-04T20:28:01,196 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa54468220064b4a8e81eb1e76faef31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1733344076900 2024-12-04T20:28:01,196 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee85ecf46dc464e84064a6ed363e801, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1733344077551 2024-12-04T20:28:01,196 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83873d089999406da66f7fa68af29e05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=896, earliestPutTs=1733344078754 2024-12-04T20:28:01,204 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4d991821bf5842b6ec138d6f2a184fc#C#compaction#104 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:01,204 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/ec9f0eacd9ad4f6aafb273d4eeb54243 is 50, key is test_row_0/C:col10/1733344079904/Put/seqid=0 2024-12-04T20:28:01,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741941_1117 (size=13459) 2024-12-04T20:28:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T20:28:01,302 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 155 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 171 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2523 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2442 2024-12-04T20:28:01,302 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T20:28:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1197 2024-12-04T20:28:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3578 rows 2024-12-04T20:28:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-12-04T20:28:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3531 rows 2024-12-04T20:28:01,303 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:28:01,303 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:28:01,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:01,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:01,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:01,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T20:28:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T20:28:01,313 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344081313"}]},"ts":"1733344081313"} 2024-12-04T20:28:01,315 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T20:28:01,315 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T20:28:01,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:28:01,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, UNASSIGN}] 2024-12-04T20:28:01,321 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, UNASSIGN 2024-12-04T20:28:01,322 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e4d991821bf5842b6ec138d6f2a184fc, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:01,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, UNASSIGN because future has completed 2024-12-04T20:28:01,324 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:28:01,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:28:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T20:28:01,480 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:28:01,480 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:28:01,480 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing e4d991821bf5842b6ec138d6f2a184fc, disabling compactions & flushes 2024-12-04T20:28:01,481 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,592 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/B/b5e3067f036c40428e2a49269f1914b2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b5e3067f036c40428e2a49269f1914b2 2024-12-04T20:28:01,597 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/B of e4d991821bf5842b6ec138d6f2a184fc into b5e3067f036c40428e2a49269f1914b2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:01,597 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:28:01,597 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/B, priority=12, startTime=1733344081164; duration=0sec 2024-12-04T20:28:01,597 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:01,597 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:B 2024-12-04T20:28:01,612 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/.tmp/C/ec9f0eacd9ad4f6aafb273d4eeb54243 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ec9f0eacd9ad4f6aafb273d4eeb54243 2024-12-04T20:28:01,618 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4d991821bf5842b6ec138d6f2a184fc/C of e4d991821bf5842b6ec138d6f2a184fc into ec9f0eacd9ad4f6aafb273d4eeb54243(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:01,618 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4d991821bf5842b6ec138d6f2a184fc: 2024-12-04T20:28:01,618 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc., storeName=e4d991821bf5842b6ec138d6f2a184fc/C, priority=12, startTime=1733344081164; duration=0sec 2024-12-04T20:28:01,618 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,618 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,618 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:01,618 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4d991821bf5842b6ec138d6f2a184fc:C 2024-12-04T20:28:01,618 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. after waiting 0 ms 2024-12-04T20:28:01,618 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/88a6d8882e6c46ca9920b2fee543a323, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/44daa267c1aa41b0ae9f90d5d9ff934e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6121de962bd042eda6a934484be589bc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8e712a839d5741e18cbdd77f7e4f2bb5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/92536861da054c89bcdd3ecf601e756c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0f582b1f33354a92a0417cf273ffc91b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/bef200616d3148a1a16af8c81a4b0922, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1ae52f77548a497eb96309a66e315a0c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5a9307d1317e4122ab7f9a19e0fe9184, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/ed909a66e53b4ac9ae58c45f8fb3e006, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/b5be0033da504a4494e5188f40f5646f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2b661786ffee4a5da39b37a3ba470800, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/d9fd230e17624d5fb37609cf328c6ffd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/9142230c5220412ba8579cbc92afbef2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/3624d67e9dc947958e3bb50bbd83270e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/da63ccddad054713af4377b99cbe464e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e86a220f58ef476985198c6f8535b672, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e448e6c192e54b12ba484d4713411170, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/38515607bd53431092b099ba0d819b92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0bb6719895b348349896dc41653d1757, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e8d94c554e4a4f33a8da3932a077de0b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/638271dacf3347b19f1ef7cfb8727266, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/72fbd44c39e24131acbf21fba348ed68, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/42d0793463c542ddaa08e09d1a4cc21a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/caa66c089d434998bdb88122c067a8d5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8a1d35cec6154e799216ed0c7dcc0a7c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/81a9117d0cbf4849ab384ad7789d187e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/65e5d14619034478b56f3742413d15cf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0357d2dc36b4499ab471ec7c7cb6fdc9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1cede52a9c4b4485b23f51c55bd680e0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5852dbfc965547feb3fb37cbd4aa2d00, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6661dbf0f74e45b7bf68c821896e3513, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5cb55e240ad8464fba094364b0fd8612] to archive 2024-12-04T20:28:01,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:28:01,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/88a6d8882e6c46ca9920b2fee543a323 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/88a6d8882e6c46ca9920b2fee543a323 2024-12-04T20:28:01,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/44daa267c1aa41b0ae9f90d5d9ff934e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/44daa267c1aa41b0ae9f90d5d9ff934e 2024-12-04T20:28:01,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6121de962bd042eda6a934484be589bc to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6121de962bd042eda6a934484be589bc 2024-12-04T20:28:01,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8e712a839d5741e18cbdd77f7e4f2bb5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8e712a839d5741e18cbdd77f7e4f2bb5 2024-12-04T20:28:01,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/92536861da054c89bcdd3ecf601e756c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/92536861da054c89bcdd3ecf601e756c 2024-12-04T20:28:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T20:28:01,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0f582b1f33354a92a0417cf273ffc91b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0f582b1f33354a92a0417cf273ffc91b 2024-12-04T20:28:01,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/bef200616d3148a1a16af8c81a4b0922 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/bef200616d3148a1a16af8c81a4b0922 2024-12-04T20:28:01,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1ae52f77548a497eb96309a66e315a0c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1ae52f77548a497eb96309a66e315a0c 2024-12-04T20:28:01,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5a9307d1317e4122ab7f9a19e0fe9184 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5a9307d1317e4122ab7f9a19e0fe9184 2024-12-04T20:28:01,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/ed909a66e53b4ac9ae58c45f8fb3e006 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/ed909a66e53b4ac9ae58c45f8fb3e006 2024-12-04T20:28:01,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/b5be0033da504a4494e5188f40f5646f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/b5be0033da504a4494e5188f40f5646f 2024-12-04T20:28:01,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2b661786ffee4a5da39b37a3ba470800 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2b661786ffee4a5da39b37a3ba470800 2024-12-04T20:28:01,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/d9fd230e17624d5fb37609cf328c6ffd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/d9fd230e17624d5fb37609cf328c6ffd 2024-12-04T20:28:01,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/9142230c5220412ba8579cbc92afbef2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/9142230c5220412ba8579cbc92afbef2 2024-12-04T20:28:01,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/3624d67e9dc947958e3bb50bbd83270e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/3624d67e9dc947958e3bb50bbd83270e 2024-12-04T20:28:01,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/da63ccddad054713af4377b99cbe464e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/da63ccddad054713af4377b99cbe464e 2024-12-04T20:28:01,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e86a220f58ef476985198c6f8535b672 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e86a220f58ef476985198c6f8535b672 2024-12-04T20:28:01,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e448e6c192e54b12ba484d4713411170 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e448e6c192e54b12ba484d4713411170 2024-12-04T20:28:01,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/38515607bd53431092b099ba0d819b92 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/38515607bd53431092b099ba0d819b92 2024-12-04T20:28:01,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0bb6719895b348349896dc41653d1757 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0bb6719895b348349896dc41653d1757 2024-12-04T20:28:01,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e8d94c554e4a4f33a8da3932a077de0b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/e8d94c554e4a4f33a8da3932a077de0b 2024-12-04T20:28:01,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/638271dacf3347b19f1ef7cfb8727266 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/638271dacf3347b19f1ef7cfb8727266 2024-12-04T20:28:01,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/72fbd44c39e24131acbf21fba348ed68 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/72fbd44c39e24131acbf21fba348ed68 2024-12-04T20:28:01,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/42d0793463c542ddaa08e09d1a4cc21a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/42d0793463c542ddaa08e09d1a4cc21a 2024-12-04T20:28:01,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/caa66c089d434998bdb88122c067a8d5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/caa66c089d434998bdb88122c067a8d5 2024-12-04T20:28:01,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8a1d35cec6154e799216ed0c7dcc0a7c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/8a1d35cec6154e799216ed0c7dcc0a7c 2024-12-04T20:28:01,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/81a9117d0cbf4849ab384ad7789d187e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/81a9117d0cbf4849ab384ad7789d187e 2024-12-04T20:28:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/65e5d14619034478b56f3742413d15cf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/65e5d14619034478b56f3742413d15cf 2024-12-04T20:28:01,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0357d2dc36b4499ab471ec7c7cb6fdc9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/0357d2dc36b4499ab471ec7c7cb6fdc9 2024-12-04T20:28:01,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1cede52a9c4b4485b23f51c55bd680e0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/1cede52a9c4b4485b23f51c55bd680e0 2024-12-04T20:28:01,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5852dbfc965547feb3fb37cbd4aa2d00 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5852dbfc965547feb3fb37cbd4aa2d00 2024-12-04T20:28:01,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6661dbf0f74e45b7bf68c821896e3513 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6661dbf0f74e45b7bf68c821896e3513 2024-12-04T20:28:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/6a7541e9ef6b4dc6a6d7c8ef5f679f65 2024-12-04T20:28:01,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5cb55e240ad8464fba094364b0fd8612 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/5cb55e240ad8464fba094364b0fd8612 2024-12-04T20:28:01,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b992e8ca0c3440e68d638dadf6c0a9e0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a93b35cd6c5d414696a31f42eec91f9f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/174c40588db743aa891eaf583c6288bf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/48944c6fab23441db6a5369f1d352143, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ffcc266af1c54fa796242eeb97855149, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f421c046723248a0b8742c4cd7783f15, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/3d57b57411694ce18ddcfe2bd9724ee1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/23e8c0df73824163bc9cf56c9a863ad6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/99011fa8c0e24987ba62a27865046443, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9db4387a1ea346e6820a28c19fca921d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/bb4c4260ea8b465ba236ad5a5e6277b3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/349952afd52d419f9b5bb8d66eb08b43, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/abb05c32516a4e96895ee7b26fe259b2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a4a154985ac24566bf59338a37b4e4d1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f3e9e4754fc64a29ae34a829b580aaff, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/55c438a8ac544448a7be3e652a6f937c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2abc7bcb51c547f483969b257c14b781, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73475dd4aa024b0ca29be02fd344348d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/7e331e64429b41e4bae5002420da3c8f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2e4af94205c74ceb9bbee176b1859087, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/8c5c8fd70fa5410ebd9ac925f124ee7c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ad48be111cbd4b7eb24b08a3b990803f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60b707a58b8c44e8a2748ef0f7bc131f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/692b6dc4cf9e4aa08bf37204363a1e6d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60774f7db88a417e9db704427c2cfe60, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73efbe5d6de14590b9c809376b2dccd5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/c680fd5174a14699b36edd748cd13e9c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9925e8a44b9d4295a9ea3f18df68941e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f41a47e6b95743bb87e848ce2da32233, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/e9e76fc9b3d54761b9e2b4566b9bfbcf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2ebfd866b05a4e7f941c20b0317253c8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/fff7a25bb2644d53a66fa8ab229fb9a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/38ce3f8d82154b7f8c8c656c714ac8e1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/77029b6190c04f0fa979ace9c8796c0d] to archive 2024-12-04T20:28:01,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:28:01,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b992e8ca0c3440e68d638dadf6c0a9e0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b992e8ca0c3440e68d638dadf6c0a9e0 2024-12-04T20:28:01,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a93b35cd6c5d414696a31f42eec91f9f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a93b35cd6c5d414696a31f42eec91f9f 2024-12-04T20:28:01,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/174c40588db743aa891eaf583c6288bf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/174c40588db743aa891eaf583c6288bf 2024-12-04T20:28:01,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/48944c6fab23441db6a5369f1d352143 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/48944c6fab23441db6a5369f1d352143 2024-12-04T20:28:01,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ffcc266af1c54fa796242eeb97855149 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ffcc266af1c54fa796242eeb97855149 2024-12-04T20:28:01,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f421c046723248a0b8742c4cd7783f15 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f421c046723248a0b8742c4cd7783f15 2024-12-04T20:28:01,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/3d57b57411694ce18ddcfe2bd9724ee1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/3d57b57411694ce18ddcfe2bd9724ee1 2024-12-04T20:28:01,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/23e8c0df73824163bc9cf56c9a863ad6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/23e8c0df73824163bc9cf56c9a863ad6 2024-12-04T20:28:01,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/99011fa8c0e24987ba62a27865046443 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/99011fa8c0e24987ba62a27865046443 2024-12-04T20:28:01,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9db4387a1ea346e6820a28c19fca921d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9db4387a1ea346e6820a28c19fca921d 2024-12-04T20:28:01,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/bb4c4260ea8b465ba236ad5a5e6277b3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/bb4c4260ea8b465ba236ad5a5e6277b3 2024-12-04T20:28:01,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/349952afd52d419f9b5bb8d66eb08b43 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/349952afd52d419f9b5bb8d66eb08b43 2024-12-04T20:28:01,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/abb05c32516a4e96895ee7b26fe259b2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/abb05c32516a4e96895ee7b26fe259b2 2024-12-04T20:28:01,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a4a154985ac24566bf59338a37b4e4d1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/a4a154985ac24566bf59338a37b4e4d1 2024-12-04T20:28:01,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f3e9e4754fc64a29ae34a829b580aaff to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f3e9e4754fc64a29ae34a829b580aaff 2024-12-04T20:28:01,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/55c438a8ac544448a7be3e652a6f937c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/55c438a8ac544448a7be3e652a6f937c 2024-12-04T20:28:01,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2abc7bcb51c547f483969b257c14b781 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2abc7bcb51c547f483969b257c14b781 2024-12-04T20:28:01,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73475dd4aa024b0ca29be02fd344348d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73475dd4aa024b0ca29be02fd344348d 2024-12-04T20:28:01,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/7e331e64429b41e4bae5002420da3c8f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/7e331e64429b41e4bae5002420da3c8f 2024-12-04T20:28:01,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2e4af94205c74ceb9bbee176b1859087 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2e4af94205c74ceb9bbee176b1859087 2024-12-04T20:28:01,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/8c5c8fd70fa5410ebd9ac925f124ee7c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/8c5c8fd70fa5410ebd9ac925f124ee7c 2024-12-04T20:28:01,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ad48be111cbd4b7eb24b08a3b990803f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/ad48be111cbd4b7eb24b08a3b990803f 2024-12-04T20:28:01,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60b707a58b8c44e8a2748ef0f7bc131f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60b707a58b8c44e8a2748ef0f7bc131f 2024-12-04T20:28:01,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/692b6dc4cf9e4aa08bf37204363a1e6d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/692b6dc4cf9e4aa08bf37204363a1e6d 2024-12-04T20:28:01,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60774f7db88a417e9db704427c2cfe60 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/60774f7db88a417e9db704427c2cfe60 2024-12-04T20:28:01,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73efbe5d6de14590b9c809376b2dccd5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/73efbe5d6de14590b9c809376b2dccd5 2024-12-04T20:28:01,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/c680fd5174a14699b36edd748cd13e9c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/c680fd5174a14699b36edd748cd13e9c 2024-12-04T20:28:01,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9925e8a44b9d4295a9ea3f18df68941e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/9925e8a44b9d4295a9ea3f18df68941e 2024-12-04T20:28:01,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f41a47e6b95743bb87e848ce2da32233 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/f41a47e6b95743bb87e848ce2da32233 2024-12-04T20:28:01,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/e9e76fc9b3d54761b9e2b4566b9bfbcf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/e9e76fc9b3d54761b9e2b4566b9bfbcf 2024-12-04T20:28:01,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2ebfd866b05a4e7f941c20b0317253c8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/2ebfd866b05a4e7f941c20b0317253c8 2024-12-04T20:28:01,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/fff7a25bb2644d53a66fa8ab229fb9a4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/fff7a25bb2644d53a66fa8ab229fb9a4 2024-12-04T20:28:01,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/38ce3f8d82154b7f8c8c656c714ac8e1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/38ce3f8d82154b7f8c8c656c714ac8e1 2024-12-04T20:28:01,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/77029b6190c04f0fa979ace9c8796c0d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/77029b6190c04f0fa979ace9c8796c0d 2024-12-04T20:28:01,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/a95fdab46c84417093f66653bc31c057, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/c1dc31eff59946b9a2e71ee17fe8c7f6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6b2130f68397411ea5373581d4668fd6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/213bf6faa5a94657bc5ff0afed0fbc62, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f499868ac0be4af1bcd58524e795de49, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6cda30e7995741c2a43d70183f68a6e4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/bd6c5019439746c7a9751663d97c5510, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e45328019b434bc6bf655b717ec131c6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/7cc8dd59a6c04a94a539418372d782fe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e254218eaf794ab5b9376ecc9491d2e1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/eba9b7fa6ff04e8f8442c2f1a0d52511, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/deb2b064ea264f669f316be78a93ed1d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d76f564a70b04372b4437db98a9a33f5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cdc7266631bb4b6eb3738a12133bf307, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/9a602bdaebcc4af7a700144a79070e22, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/71eecdf71c0640f6a7c6e89e68afd86d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/19c023100cea4bb3b068ad85b9af2b02, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/b319ac8a0009466b8270684259773137, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/77f7ce872fb54e4fbc490e83d123bd4d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d44428ec51084934ac96d72619426b3a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/4d32ace258b24bf38a639551883e5cfe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/8329860e3f9543779d06069a1507ba2a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ad02adf8a5214628a1c61165e1c7708c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f1cf400146f2409b882f1fb315d87d70, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2464a02f149a436bb19ac93d22cbb49a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/28de17b30cb849c98de67503d23a5eb8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f7f45f0e1559463ea0dd25951a8d3345, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2ad7264ec1fc400faacff40b09b0e33f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/69897ac88c9f43369a3b289f540f63fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cb052562c89040c2b7630fef1380ab9d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/16f4d8b7ffc04483b87ecbcd00277dbb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/aa54468220064b4a8e81eb1e76faef31, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/0ee85ecf46dc464e84064a6ed363e801, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/83873d089999406da66f7fa68af29e05] to archive 2024-12-04T20:28:01,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:28:01,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/a95fdab46c84417093f66653bc31c057 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/a95fdab46c84417093f66653bc31c057 2024-12-04T20:28:01,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/c1dc31eff59946b9a2e71ee17fe8c7f6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/c1dc31eff59946b9a2e71ee17fe8c7f6 2024-12-04T20:28:01,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6b2130f68397411ea5373581d4668fd6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6b2130f68397411ea5373581d4668fd6 2024-12-04T20:28:01,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/213bf6faa5a94657bc5ff0afed0fbc62 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/213bf6faa5a94657bc5ff0afed0fbc62 2024-12-04T20:28:01,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f499868ac0be4af1bcd58524e795de49 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f499868ac0be4af1bcd58524e795de49 2024-12-04T20:28:01,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6cda30e7995741c2a43d70183f68a6e4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/6cda30e7995741c2a43d70183f68a6e4 2024-12-04T20:28:01,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/bd6c5019439746c7a9751663d97c5510 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/bd6c5019439746c7a9751663d97c5510 2024-12-04T20:28:01,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e45328019b434bc6bf655b717ec131c6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e45328019b434bc6bf655b717ec131c6 2024-12-04T20:28:01,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/7cc8dd59a6c04a94a539418372d782fe to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/7cc8dd59a6c04a94a539418372d782fe 2024-12-04T20:28:01,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e254218eaf794ab5b9376ecc9491d2e1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/e254218eaf794ab5b9376ecc9491d2e1 2024-12-04T20:28:01,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/eba9b7fa6ff04e8f8442c2f1a0d52511 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/eba9b7fa6ff04e8f8442c2f1a0d52511 2024-12-04T20:28:01,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/deb2b064ea264f669f316be78a93ed1d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/deb2b064ea264f669f316be78a93ed1d 2024-12-04T20:28:01,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d76f564a70b04372b4437db98a9a33f5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d76f564a70b04372b4437db98a9a33f5 2024-12-04T20:28:01,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cdc7266631bb4b6eb3738a12133bf307 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cdc7266631bb4b6eb3738a12133bf307 2024-12-04T20:28:01,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/9a602bdaebcc4af7a700144a79070e22 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/9a602bdaebcc4af7a700144a79070e22 2024-12-04T20:28:01,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/71eecdf71c0640f6a7c6e89e68afd86d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/71eecdf71c0640f6a7c6e89e68afd86d 2024-12-04T20:28:01,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/19c023100cea4bb3b068ad85b9af2b02 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/19c023100cea4bb3b068ad85b9af2b02 2024-12-04T20:28:01,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/b319ac8a0009466b8270684259773137 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/b319ac8a0009466b8270684259773137 2024-12-04T20:28:01,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/77f7ce872fb54e4fbc490e83d123bd4d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/77f7ce872fb54e4fbc490e83d123bd4d 2024-12-04T20:28:01,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d44428ec51084934ac96d72619426b3a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/d44428ec51084934ac96d72619426b3a 2024-12-04T20:28:01,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/4d32ace258b24bf38a639551883e5cfe to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/4d32ace258b24bf38a639551883e5cfe 2024-12-04T20:28:01,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/8329860e3f9543779d06069a1507ba2a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/8329860e3f9543779d06069a1507ba2a 2024-12-04T20:28:01,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ad02adf8a5214628a1c61165e1c7708c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ad02adf8a5214628a1c61165e1c7708c 2024-12-04T20:28:01,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f1cf400146f2409b882f1fb315d87d70 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f1cf400146f2409b882f1fb315d87d70 2024-12-04T20:28:01,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2464a02f149a436bb19ac93d22cbb49a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2464a02f149a436bb19ac93d22cbb49a 2024-12-04T20:28:01,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/28de17b30cb849c98de67503d23a5eb8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/28de17b30cb849c98de67503d23a5eb8 2024-12-04T20:28:01,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f7f45f0e1559463ea0dd25951a8d3345 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/f7f45f0e1559463ea0dd25951a8d3345 2024-12-04T20:28:01,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2ad7264ec1fc400faacff40b09b0e33f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/2ad7264ec1fc400faacff40b09b0e33f 2024-12-04T20:28:01,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/69897ac88c9f43369a3b289f540f63fd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/69897ac88c9f43369a3b289f540f63fd 2024-12-04T20:28:01,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cb052562c89040c2b7630fef1380ab9d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/cb052562c89040c2b7630fef1380ab9d 2024-12-04T20:28:01,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/16f4d8b7ffc04483b87ecbcd00277dbb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/16f4d8b7ffc04483b87ecbcd00277dbb 2024-12-04T20:28:01,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/aa54468220064b4a8e81eb1e76faef31 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/aa54468220064b4a8e81eb1e76faef31 2024-12-04T20:28:01,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/0ee85ecf46dc464e84064a6ed363e801 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/0ee85ecf46dc464e84064a6ed363e801 2024-12-04T20:28:01,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/83873d089999406da66f7fa68af29e05 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/83873d089999406da66f7fa68af29e05 2024-12-04T20:28:01,761 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/recovered.edits/902.seqid, newMaxSeqId=902, maxSeqId=1 2024-12-04T20:28:01,761 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc. 2024-12-04T20:28:01,762 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for e4d991821bf5842b6ec138d6f2a184fc: Waiting for close lock at 1733344081480Running coprocessor pre-close hooks at 1733344081480Disabling compacts and flushes for region at 1733344081480Disabling writes for close at 1733344081618 (+138 ms)Writing region close event to WAL at 1733344081756 (+138 ms)Running coprocessor post-close hooks at 1733344081761 (+5 ms)Closed at 1733344081761 2024-12-04T20:28:01,764 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:28:01,765 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e4d991821bf5842b6ec138d6f2a184fc, regionState=CLOSED 2024-12-04T20:28:01,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:28:01,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-04T20:28:01,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure e4d991821bf5842b6ec138d6f2a184fc, server=c2ef38372881,45443,1733344054291 in 444 msec 2024-12-04T20:28:01,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-04T20:28:01,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4d991821bf5842b6ec138d6f2a184fc, UNASSIGN in 450 msec 2024-12-04T20:28:01,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-04T20:28:01,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 456 msec 2024-12-04T20:28:01,775 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344081775"}]},"ts":"1733344081775"} 2024-12-04T20:28:01,777 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T20:28:01,777 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T20:28:01,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 470 msec 2024-12-04T20:28:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T20:28:01,942 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:01,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T20:28:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:01,949 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T20:28:01,950 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:01,955 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:28:01,958 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/recovered.edits] 2024-12-04T20:28:01,961 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2dae9c2453b14e43958030e399fb321e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/A/2dae9c2453b14e43958030e399fb321e 2024-12-04T20:28:01,964 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b5e3067f036c40428e2a49269f1914b2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/B/b5e3067f036c40428e2a49269f1914b2 2024-12-04T20:28:01,966 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ec9f0eacd9ad4f6aafb273d4eeb54243 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/C/ec9f0eacd9ad4f6aafb273d4eeb54243 2024-12-04T20:28:01,969 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/recovered.edits/902.seqid to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc/recovered.edits/902.seqid 2024-12-04T20:28:01,970 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/e4d991821bf5842b6ec138d6f2a184fc 2024-12-04T20:28:01,970 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T20:28:01,974 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:01,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-04T20:28:01,985 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T20:28:02,024 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T20:28:02,026 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:02,026 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T20:28:02,026 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733344082026"}]},"ts":"9223372036854775807"} 2024-12-04T20:28:02,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T20:28:02,032 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e4d991821bf5842b6ec138d6f2a184fc, NAME => 'TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T20:28:02,032 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T20:28:02,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733344082032"}]},"ts":"9223372036854775807"} 2024-12-04T20:28:02,036 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T20:28:02,037 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:02,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 92 msec 2024-12-04T20:28:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T20:28:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T20:28:02,053 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:02,069 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c2ef38372881:45443-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_862031392_22 at /127.0.0.1:48864 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c2ef38372881:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1924289665_22 at /127.0.0.1:53146 [Waiting for operation #231] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1924289665_22 at /127.0.0.1:34428 [Waiting for operation #217] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_862031392_22 at /127.0.0.1:34436 [Waiting for operation #215] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=458 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4056 (was 4741) 2024-12-04T20:28:02,081 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=458, ProcessCount=11, AvailableMemoryMB=4055 2024-12-04T20:28:02,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:28:02,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:28:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:02,086 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:28:02,086 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:02,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-04T20:28:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T20:28:02,087 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:28:02,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741942_1118 (size=1061) 2024-12-04T20:28:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T20:28:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T20:28:02,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:28:02,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741943_1119 (size=53) 2024-12-04T20:28:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T20:28:02,786 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:28:02,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:28:02,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8f15724cf3b6cc25ca4bfa788ed7a570, disabling compactions & flushes 2024-12-04T20:28:02,904 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:02,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:02,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. after waiting 0 ms 2024-12-04T20:28:02,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:02,904 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:02,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f15724cf3b6cc25ca4bfa788ed7a570: Waiting for close lock at 1733344082904Disabling compacts and flushes for region at 1733344082904Disabling writes for close at 1733344082904Writing region close event to WAL at 1733344082904Closed at 1733344082904 2024-12-04T20:28:02,906 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:28:02,906 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733344082906"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733344082906"}]},"ts":"1733344082906"} 2024-12-04T20:28:02,908 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:28:02,909 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:28:02,909 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344082909"}]},"ts":"1733344082909"} 2024-12-04T20:28:02,910 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T20:28:02,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, ASSIGN}] 2024-12-04T20:28:02,912 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, ASSIGN 2024-12-04T20:28:02,912 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, ASSIGN; state=OFFLINE, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=false 2024-12-04T20:28:03,063 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:03,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, ASSIGN because future has completed 2024-12-04T20:28:03,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:28:03,220 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:03,220 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:28:03,220 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,220 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:28:03,221 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,221 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T20:28:03,222 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,223 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:03,224 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f15724cf3b6cc25ca4bfa788ed7a570 columnFamilyName A 2024-12-04T20:28:03,224 DEBUG [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:03,224 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(327): Store=8f15724cf3b6cc25ca4bfa788ed7a570/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:03,224 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,225 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:03,226 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f15724cf3b6cc25ca4bfa788ed7a570 columnFamilyName B 2024-12-04T20:28:03,226 DEBUG [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:03,226 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(327): Store=8f15724cf3b6cc25ca4bfa788ed7a570/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:03,226 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,227 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:03,227 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f15724cf3b6cc25ca4bfa788ed7a570 columnFamilyName C 2024-12-04T20:28:03,227 DEBUG [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:03,228 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(327): Store=8f15724cf3b6cc25ca4bfa788ed7a570/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:03,228 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:03,228 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,229 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,229 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:28:03,231 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,233 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:28:03,233 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 8f15724cf3b6cc25ca4bfa788ed7a570; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63374907, jitterRate=-0.05564029514789581}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:28:03,233 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:03,234 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 8f15724cf3b6cc25ca4bfa788ed7a570: Running coprocessor pre-open hook at 1733344083221Writing region info on filesystem at 1733344083221Initializing all the Stores at 1733344083222 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344083222Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344083222Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344083222Cleaning up temporary data from old regions at 1733344083230 (+8 ms)Running coprocessor post-open hooks at 1733344083233 (+3 ms)Region opened successfully at 1733344083234 (+1 ms) 2024-12-04T20:28:03,235 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., pid=30, masterSystemTime=1733344083217 2024-12-04T20:28:03,237 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:03,237 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:03,237 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:03,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:28:03,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T20:28:03,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 in 176 msec 2024-12-04T20:28:03,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-04T20:28:03,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, ASSIGN in 333 msec 2024-12-04T20:28:03,245 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:28:03,245 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344083245"}]},"ts":"1733344083245"} 2024-12-04T20:28:03,247 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T20:28:03,248 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:28:03,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-12-04T20:28:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T20:28:04,231 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:04,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:04,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:04,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:04,234 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51362aa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:04,236 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccf8029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:04,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:04,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:04,240 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:28:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,43507,1733344053664 2024-12-04T20:28:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@227cf768 2024-12-04T20:28:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:28:04,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:28:04,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:28:04,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:28:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:04,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741944_1120 (size=1097) 2024-12-04T20:28:04,668 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T20:28:04,668 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T20:28:04,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:28:04,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, REOPEN/MOVE}] 2024-12-04T20:28:04,678 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, REOPEN/MOVE 2024-12-04T20:28:04,679 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:04,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, REOPEN/MOVE because future has completed 2024-12-04T20:28:04,681 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:28:04,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:28:04,834 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:04,834 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:28:04,834 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 8f15724cf3b6cc25ca4bfa788ed7a570, disabling compactions & flushes 2024-12-04T20:28:04,834 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:04,834 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:04,834 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. after waiting 0 ms 2024-12-04T20:28:04,834 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:04,838 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T20:28:04,839 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:04,839 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 8f15724cf3b6cc25ca4bfa788ed7a570: Waiting for close lock at 1733344084834Running coprocessor pre-close hooks at 1733344084834Disabling compacts and flushes for region at 1733344084834Disabling writes for close at 1733344084834Writing region close event to WAL at 1733344084835 (+1 ms)Running coprocessor post-close hooks at 1733344084838 (+3 ms)Closed at 1733344084839 (+1 ms) 2024-12-04T20:28:04,839 WARN [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 8f15724cf3b6cc25ca4bfa788ed7a570 to self. 2024-12-04T20:28:04,840 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:04,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=CLOSED 2024-12-04T20:28:04,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:28:04,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-04T20:28:04,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 in 162 msec 2024-12-04T20:28:04,845 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, REOPEN/MOVE; state=CLOSED, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=true 2024-12-04T20:28:04,996 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:04,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, REOPEN/MOVE because future has completed 2024-12-04T20:28:04,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:28:05,152 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,153 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:28:05,153 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,153 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:28:05,153 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,153 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,155 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,155 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:05,159 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f15724cf3b6cc25ca4bfa788ed7a570 columnFamilyName A 2024-12-04T20:28:05,161 DEBUG [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:05,161 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(327): Store=8f15724cf3b6cc25ca4bfa788ed7a570/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:05,162 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,163 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:05,163 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f15724cf3b6cc25ca4bfa788ed7a570 columnFamilyName B 2024-12-04T20:28:05,163 DEBUG [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:05,163 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(327): Store=8f15724cf3b6cc25ca4bfa788ed7a570/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:05,164 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,164 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:05,164 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f15724cf3b6cc25ca4bfa788ed7a570 columnFamilyName C 2024-12-04T20:28:05,164 DEBUG [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:05,165 INFO [StoreOpener-8f15724cf3b6cc25ca4bfa788ed7a570-1 {}] regionserver.HStore(327): Store=8f15724cf3b6cc25ca4bfa788ed7a570/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:05,165 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,165 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,165 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,166 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,167 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,167 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,167 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:28:05,168 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,169 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 8f15724cf3b6cc25ca4bfa788ed7a570; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70911389, jitterRate=0.05666203796863556}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:28:05,169 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,170 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 8f15724cf3b6cc25ca4bfa788ed7a570: Running coprocessor pre-open hook at 1733344085153Writing region info on filesystem at 1733344085153Initializing all the Stores at 1733344085154 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344085154Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344085154Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344085154Cleaning up temporary data from old regions at 1733344085167 (+13 ms)Running coprocessor post-open hooks at 1733344085169 (+2 ms)Region opened successfully at 1733344085170 (+1 ms) 2024-12-04T20:28:05,171 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., pid=35, masterSystemTime=1733344085149 2024-12-04T20:28:05,173 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,173 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,173 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=OPEN, openSeqNum=5, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:28:05,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-04T20:28:05,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 in 178 msec 2024-12-04T20:28:05,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-04T20:28:05,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, REOPEN/MOVE in 499 msec 2024-12-04T20:28:05,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-04T20:28:05,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-12-04T20:28:05,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 936 msec 2024-12-04T20:28:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T20:28:05,187 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:05,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,188 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a55aece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,189 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b42a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,191 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,193 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@360d2612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,195 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b21c481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,196 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30338289, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,199 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79dff5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,200 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b709593, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,202 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4559428c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,204 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552f7e90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,206 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b79170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3720fcac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,210 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60847fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,212 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,214 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d89879e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,215 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22644b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:05,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:05,217 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34822828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:05,219 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12988e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:05,220 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,221 DEBUG [Thread-618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,233 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-04T20:28:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T20:28:05,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:28:05,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:05,238 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:05,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:05,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:05,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:05,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:05,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:05,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:05,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:05,245 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,269 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,270 DEBUG [Thread-621 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,271 DEBUG [Thread-621 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344145268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,272 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344085258, max timestamp=1733344085267Num uniques 859; 2024-12-04T20:28:05,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344085258, max timestamp=1733344085267Num uniques 859; 2024-12-04T20:28:05,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:05,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:05,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,273 DEBUG [Thread-610 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344085258, max timestamp=1733344085267Num uniques 859; 2024-12-04T20:28:05,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:05,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,281 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,301 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,313 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:05,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120480b224e9642f4940b66be7914bae3158_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344085237/Put/seqid=0 2024-12-04T20:28:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:05,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:28:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344145324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344145324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344145325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344145328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741945_1121 (size=12154) 2024-12-04T20:28:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T20:28:05,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344145381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:05,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:05,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344145441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344145442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344145442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344145442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:05,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T20:28:05,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344145592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344145652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344145652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344145661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344145652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:05,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:05,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,741 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:05,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120480b224e9642f4940b66be7914bae3158_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120480b224e9642f4940b66be7914bae3158_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:05,748 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c28f45bd5c7b406492e2da9c81b66b8c, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:05,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c28f45bd5c7b406492e2da9c81b66b8c is 175, key is test_row_0/A:col10/1733344085237/Put/seqid=0 2024-12-04T20:28:05,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741946_1122 (size=30955) 2024-12-04T20:28:05,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c28f45bd5c7b406492e2da9c81b66b8c 2024-12-04T20:28:05,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d2f17319f9c84e9d9be67e4f9f07a98f is 50, key is test_row_0/B:col10/1733344085237/Put/seqid=0 2024-12-04T20:28:05,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:05,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T20:28:05,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741947_1123 (size=12001) 2024-12-04T20:28:05,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344145902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344145972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344145972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:05,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344145982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344146001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:06,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:06,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d2f17319f9c84e9d9be67e4f9f07a98f 2024-12-04T20:28:06,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:06,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/f69c0dbef35f44aeb552ae842282a868 is 50, key is test_row_0/C:col10/1733344085237/Put/seqid=0 2024-12-04T20:28:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T20:28:06,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741948_1124 (size=12001) 2024-12-04T20:28:06,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/f69c0dbef35f44aeb552ae842282a868 2024-12-04T20:28:06,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c28f45bd5c7b406492e2da9c81b66b8c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c28f45bd5c7b406492e2da9c81b66b8c 2024-12-04T20:28:06,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c28f45bd5c7b406492e2da9c81b66b8c, entries=150, sequenceid=15, filesize=30.2 K 2024-12-04T20:28:06,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d2f17319f9c84e9d9be67e4f9f07a98f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d2f17319f9c84e9d9be67e4f9f07a98f 2024-12-04T20:28:06,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d2f17319f9c84e9d9be67e4f9f07a98f, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T20:28:06,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/f69c0dbef35f44aeb552ae842282a868 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/f69c0dbef35f44aeb552ae842282a868 2024-12-04T20:28:06,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/f69c0dbef35f44aeb552ae842282a868, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T20:28:06,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1192ms, sequenceid=15, compaction requested=false 2024-12-04T20:28:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:06,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T20:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:06,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040fb9990bffd745db8a061437b2b3407b_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344085324/Put/seqid=0 2024-12-04T20:28:06,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:06,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:06,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344146488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344146491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344146491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344146497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344146512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741949_1125 (size=14594) 2024-12-04T20:28:06,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344146602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344146611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:06,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:06,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:06,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344146811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344146822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:06,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:06,944 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040fb9990bffd745db8a061437b2b3407b_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040fb9990bffd745db8a061437b2b3407b_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:06,946 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/77b5c9a8e2724949aaf7af1c0473c02d, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:06,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/77b5c9a8e2724949aaf7af1c0473c02d is 175, key is test_row_0/A:col10/1733344085324/Put/seqid=0 2024-12-04T20:28:06,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:06,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:06,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741950_1126 (size=39549) 2024-12-04T20:28:06,988 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/77b5c9a8e2724949aaf7af1c0473c02d 2024-12-04T20:28:07,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7107d76993e1499a8e7575a0941154c7 is 50, key is test_row_0/B:col10/1733344085324/Put/seqid=0 2024-12-04T20:28:07,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741951_1127 (size=12001) 2024-12-04T20:28:07,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7107d76993e1499a8e7575a0941154c7 2024-12-04T20:28:07,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/48844e063e6c46fe89ca23888aa5aa64 is 50, key is test_row_0/C:col10/1733344085324/Put/seqid=0 2024-12-04T20:28:07,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344147122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741952_1128 (size=12001) 2024-12-04T20:28:07,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344147141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,326 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:28:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T20:28:07,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344147502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344147502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,512 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,513 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/48844e063e6c46fe89ca23888aa5aa64 2024-12-04T20:28:07,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344147531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,537 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/77b5c9a8e2724949aaf7af1c0473c02d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/77b5c9a8e2724949aaf7af1c0473c02d 2024-12-04T20:28:07,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/77b5c9a8e2724949aaf7af1c0473c02d, entries=200, sequenceid=52, filesize=38.6 K 2024-12-04T20:28:07,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7107d76993e1499a8e7575a0941154c7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7107d76993e1499a8e7575a0941154c7 2024-12-04T20:28:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7107d76993e1499a8e7575a0941154c7, entries=150, sequenceid=52, filesize=11.7 K 2024-12-04T20:28:07,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/48844e063e6c46fe89ca23888aa5aa64 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/48844e063e6c46fe89ca23888aa5aa64 2024-12-04T20:28:07,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:07,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/48844e063e6c46fe89ca23888aa5aa64, entries=150, sequenceid=52, filesize=11.7 K 2024-12-04T20:28:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1127ms, sequenceid=52, compaction requested=false 2024-12-04T20:28:07,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:07,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T20:28:07,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:07,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:07,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:07,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:07,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:07,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204427b819cafa3449e8817cb71577c009e_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344087641/Put/seqid=0 2024-12-04T20:28:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741953_1129 (size=12154) 2024-12-04T20:28:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,706 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,711 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204427b819cafa3449e8817cb71577c009e_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204427b819cafa3449e8817cb71577c009e_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,712 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/1a5044f6941042f4a38950cbb0554730, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/1a5044f6941042f4a38950cbb0554730 is 175, key is test_row_0/A:col10/1733344087641/Put/seqid=0 2024-12-04T20:28:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:07,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741954_1130 (size=30951) 2024-12-04T20:28:07,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/1a5044f6941042f4a38950cbb0554730 2024-12-04T20:28:07,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344147782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733344087753, max timestamp=1733344087786Num uniques 859; 2024-12-04T20:28:07,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733344087753, max timestamp=1733344087786Num uniques 859; 2024-12-04T20:28:07,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T20:28:07,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T20:28:07,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733344087753, max timestamp=1733344087786Num uniques 859; 2024-12-04T20:28:07,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T20:28:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3587dfe119574f2c937102eb55fb537e is 50, key is test_row_0/B:col10/1733344087641/Put/seqid=0 2024-12-04T20:28:07,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741955_1131 (size=9657) 2024-12-04T20:28:07,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3587dfe119574f2c937102eb55fb537e 2024-12-04T20:28:07,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/1c796019afe04ae7ad611f7554fa65b2 is 50, key is test_row_0/C:col10/1733344087641/Put/seqid=0 2024-12-04T20:28:07,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344147865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:07,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:07,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741956_1132 (size=9657) 2024-12-04T20:28:07,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344147901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:07,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344147981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:08,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:08,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344148111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:08,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:08,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:08,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:08,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:08,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344148201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:08,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/1c796019afe04ae7ad611f7554fa65b2 2024-12-04T20:28:08,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/1a5044f6941042f4a38950cbb0554730 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/1a5044f6941042f4a38950cbb0554730 2024-12-04T20:28:08,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/1a5044f6941042f4a38950cbb0554730, entries=150, sequenceid=63, filesize=30.2 K 2024-12-04T20:28:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3587dfe119574f2c937102eb55fb537e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3587dfe119574f2c937102eb55fb537e 2024-12-04T20:28:08,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3587dfe119574f2c937102eb55fb537e, entries=100, sequenceid=63, filesize=9.4 K 2024-12-04T20:28:08,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/1c796019afe04ae7ad611f7554fa65b2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/1c796019afe04ae7ad611f7554fa65b2 2024-12-04T20:28:08,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/1c796019afe04ae7ad611f7554fa65b2, entries=100, sequenceid=63, filesize=9.4 K 2024-12-04T20:28:08,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 685ms, sequenceid=63, compaction requested=true 2024-12-04T20:28:08,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:08,332 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:08,332 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:08,333 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:08,333 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:08,333 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:08,333 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:08,333 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,333 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,333 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d2f17319f9c84e9d9be67e4f9f07a98f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7107d76993e1499a8e7575a0941154c7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3587dfe119574f2c937102eb55fb537e] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=32.9 K 2024-12-04T20:28:08,333 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c28f45bd5c7b406492e2da9c81b66b8c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/77b5c9a8e2724949aaf7af1c0473c02d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/1a5044f6941042f4a38950cbb0554730] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=99.1 K 2024-12-04T20:28:08,333 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,334 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f17319f9c84e9d9be67e4f9f07a98f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733344085229 2024-12-04T20:28:08,334 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c28f45bd5c7b406492e2da9c81b66b8c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/77b5c9a8e2724949aaf7af1c0473c02d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/1a5044f6941042f4a38950cbb0554730] 2024-12-04T20:28:08,334 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c28f45bd5c7b406492e2da9c81b66b8c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733344085229 2024-12-04T20:28:08,334 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7107d76993e1499a8e7575a0941154c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733344085322 2024-12-04T20:28:08,335 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3587dfe119574f2c937102eb55fb537e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344086493 2024-12-04T20:28:08,335 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77b5c9a8e2724949aaf7af1c0473c02d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733344085319 2024-12-04T20:28:08,335 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5044f6941042f4a38950cbb0554730, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344086486 2024-12-04T20:28:08,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T20:28:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,337 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-04T20:28:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:08,352 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:08,352 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/89717c6a23b24d59891960d6d326c6d5 is 50, key is test_row_0/B:col10/1733344087641/Put/seqid=0 2024-12-04T20:28:08,353 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:08,360 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120400232f47095342c9b2c9d12f1f090e66_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:08,363 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120400232f47095342c9b2c9d12f1f090e66_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:08,363 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120400232f47095342c9b2c9d12f1f090e66_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204bcd7c617f42f485cba239465963b2186_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344087863/Put/seqid=0 2024-12-04T20:28:08,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741957_1133 (size=12104) 2024-12-04T20:28:08,384 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/89717c6a23b24d59891960d6d326c6d5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/89717c6a23b24d59891960d6d326c6d5 2024-12-04T20:28:08,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741959_1135 (size=12154) 2024-12-04T20:28:08,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:08,399 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into 89717c6a23b24d59891960d6d326c6d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:08,399 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:08,399 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=13, startTime=1733344088332; duration=0sec 2024-12-04T20:28:08,399 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:08,400 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:08,400 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:08,401 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204bcd7c617f42f485cba239465963b2186_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bcd7c617f42f485cba239465963b2186_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:08,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/6beccb95ef6044c9957ddbae255a43aa, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:08,402 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:08,402 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:08,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/6beccb95ef6044c9957ddbae255a43aa is 175, key is test_row_0/A:col10/1733344087863/Put/seqid=0 2024-12-04T20:28:08,402 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:08,403 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/f69c0dbef35f44aeb552ae842282a868, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/48844e063e6c46fe89ca23888aa5aa64, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/1c796019afe04ae7ad611f7554fa65b2] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=32.9 K 2024-12-04T20:28:08,404 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f69c0dbef35f44aeb552ae842282a868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733344085229 2024-12-04T20:28:08,405 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 48844e063e6c46fe89ca23888aa5aa64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733344085322 2024-12-04T20:28:08,405 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c796019afe04ae7ad611f7554fa65b2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344086493 2024-12-04T20:28:08,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741958_1134 (size=4469) 2024-12-04T20:28:08,425 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:08,428 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/eec16a23481d4bf88b9a5e79c010b85c is 50, key is test_row_0/C:col10/1733344087641/Put/seqid=0 2024-12-04T20:28:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:08,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:08,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741960_1136 (size=30955) 2024-12-04T20:28:08,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741961_1137 (size=12104) 2024-12-04T20:28:08,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344148473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:08,479 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/eec16a23481d4bf88b9a5e79c010b85c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/eec16a23481d4bf88b9a5e79c010b85c 2024-12-04T20:28:08,489 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into eec16a23481d4bf88b9a5e79c010b85c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:08,489 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:08,489 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=13, startTime=1733344088332; duration=0sec 2024-12-04T20:28:08,489 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:08,489 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344148512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344148581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344148791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:08,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:08,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:08,813 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#115 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:08,813 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/8cdf6d19235e42a0b33a9c5ad03b1b74 is 175, key is test_row_0/A:col10/1733344087641/Put/seqid=0 2024-12-04T20:28:08,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741962_1138 (size=31165) 2024-12-04T20:28:08,843 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/8cdf6d19235e42a0b33a9c5ad03b1b74 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/8cdf6d19235e42a0b33a9c5ad03b1b74 2024-12-04T20:28:08,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/6beccb95ef6044c9957ddbae255a43aa 2024-12-04T20:28:08,852 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into 8cdf6d19235e42a0b33a9c5ad03b1b74(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:08,852 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:08,852 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=13, startTime=1733344088331; duration=0sec 2024-12-04T20:28:08,853 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:08,853 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:08,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7af2c4b1ae434aa186ac0ffef7b47ccb is 50, key is test_row_0/B:col10/1733344087863/Put/seqid=0 2024-12-04T20:28:08,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741963_1139 (size=12001) 2024-12-04T20:28:08,908 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7af2c4b1ae434aa186ac0ffef7b47ccb 2024-12-04T20:28:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/30d9dbb9896247208de8d7f10c1996d1 is 50, key is test_row_0/C:col10/1733344087863/Put/seqid=0 2024-12-04T20:28:08,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741964_1140 (size=12001) 2024-12-04T20:28:08,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/30d9dbb9896247208de8d7f10c1996d1 2024-12-04T20:28:08,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/6beccb95ef6044c9957ddbae255a43aa as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6beccb95ef6044c9957ddbae255a43aa 2024-12-04T20:28:08,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6beccb95ef6044c9957ddbae255a43aa, entries=150, sequenceid=96, filesize=30.2 K 2024-12-04T20:28:08,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7af2c4b1ae434aa186ac0ffef7b47ccb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7af2c4b1ae434aa186ac0ffef7b47ccb 2024-12-04T20:28:09,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7af2c4b1ae434aa186ac0ffef7b47ccb, entries=150, sequenceid=96, filesize=11.7 K 2024-12-04T20:28:09,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/30d9dbb9896247208de8d7f10c1996d1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/30d9dbb9896247208de8d7f10c1996d1 2024-12-04T20:28:09,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/30d9dbb9896247208de8d7f10c1996d1, entries=150, sequenceid=96, filesize=11.7 K 2024-12-04T20:28:09,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 676ms, sequenceid=96, compaction requested=false 2024-12-04T20:28:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-04T20:28:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-04T20:28:09,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-04T20:28:09,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7770 sec 2024-12-04T20:28:09,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.7860 sec 2024-12-04T20:28:09,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:28:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:09,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:09,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:09,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042edf35d14c944fef8346bb54244bad95_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344089022/Put/seqid=0 2024-12-04T20:28:09,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741965_1141 (size=14594) 2024-12-04T20:28:09,069 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:09,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042edf35d14c944fef8346bb54244bad95_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042edf35d14c944fef8346bb54244bad95_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:09,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/eac47076a9284ba7837a403cd7ccd0f4, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:09,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/eac47076a9284ba7837a403cd7ccd0f4 is 175, key is test_row_0/A:col10/1733344089022/Put/seqid=0 2024-12-04T20:28:09,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741966_1142 (size=39549) 2024-12-04T20:28:09,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344149141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344149141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344149251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344149251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T20:28:09,401 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:09,402 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-04T20:28:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T20:28:09,404 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:09,404 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:09,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:09,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344149462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344149462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T20:28:09,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/eac47076a9284ba7837a403cd7ccd0f4 2024-12-04T20:28:09,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344149521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,527 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/91bbf633aa474e36ac19ec5bb5aa48ce is 50, key is test_row_0/B:col10/1733344089022/Put/seqid=0 2024-12-04T20:28:09,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344149541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,547 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T20:28:09,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:09,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:09,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:09,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344149562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,572 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741967_1143 (size=12001) 2024-12-04T20:28:09,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T20:28:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:09,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T20:28:09,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344149772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344149772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:09,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T20:28:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:09,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:09,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/91bbf633aa474e36ac19ec5bb5aa48ce 2024-12-04T20:28:10,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/82bc80acfb2c4fe3a74e76b6c181ceab is 50, key is test_row_0/C:col10/1733344089022/Put/seqid=0 2024-12-04T20:28:10,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T20:28:10,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:10,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T20:28:10,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741968_1144 (size=12001) 2024-12-04T20:28:10,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/82bc80acfb2c4fe3a74e76b6c181ceab 2024-12-04T20:28:10,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/eac47076a9284ba7837a403cd7ccd0f4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/eac47076a9284ba7837a403cd7ccd0f4 2024-12-04T20:28:10,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/eac47076a9284ba7837a403cd7ccd0f4, entries=200, sequenceid=111, filesize=38.6 K 2024-12-04T20:28:10,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/91bbf633aa474e36ac19ec5bb5aa48ce as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/91bbf633aa474e36ac19ec5bb5aa48ce 2024-12-04T20:28:10,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/91bbf633aa474e36ac19ec5bb5aa48ce, entries=150, sequenceid=111, filesize=11.7 K 2024-12-04T20:28:10,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/82bc80acfb2c4fe3a74e76b6c181ceab as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/82bc80acfb2c4fe3a74e76b6c181ceab 2024-12-04T20:28:10,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/82bc80acfb2c4fe3a74e76b6c181ceab, entries=150, sequenceid=111, filesize=11.7 K 2024-12-04T20:28:10,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1090ms, sequenceid=111, compaction requested=true 2024-12-04T20:28:10,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:10,116 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:10,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:10,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:10,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:10,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:10,117 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:10,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:10,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:10,117 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:10,117 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:10,117 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:10,118 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/8cdf6d19235e42a0b33a9c5ad03b1b74, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6beccb95ef6044c9957ddbae255a43aa, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/eac47076a9284ba7837a403cd7ccd0f4] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=99.3 K 2024-12-04T20:28:10,118 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:10,118 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:10,118 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:10,118 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:10,118 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/8cdf6d19235e42a0b33a9c5ad03b1b74, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6beccb95ef6044c9957ddbae255a43aa, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/eac47076a9284ba7837a403cd7ccd0f4] 2024-12-04T20:28:10,121 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/89717c6a23b24d59891960d6d326c6d5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7af2c4b1ae434aa186ac0ffef7b47ccb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/91bbf633aa474e36ac19ec5bb5aa48ce] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=35.3 K 2024-12-04T20:28:10,122 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cdf6d19235e42a0b33a9c5ad03b1b74, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344085322 2024-12-04T20:28:10,122 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 89717c6a23b24d59891960d6d326c6d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344085322 2024-12-04T20:28:10,123 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af2c4b1ae434aa186ac0ffef7b47ccb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344087847 2024-12-04T20:28:10,123 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6beccb95ef6044c9957ddbae255a43aa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344087847 2024-12-04T20:28:10,124 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 91bbf633aa474e36ac19ec5bb5aa48ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733344088456 2024-12-04T20:28:10,124 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting eac47076a9284ba7837a403cd7ccd0f4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733344088456 2024-12-04T20:28:10,142 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:10,155 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:10,155 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d955e344fec14f21acba3210015f4592 is 50, key is test_row_0/B:col10/1733344089022/Put/seqid=0 2024-12-04T20:28:10,162 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204243c9e9afcb94452ab45636da098b98a_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:10,164 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204243c9e9afcb94452ab45636da098b98a_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:10,164 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204243c9e9afcb94452ab45636da098b98a_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:10,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T20:28:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:10,175 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T20:28:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:10,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741969_1145 (size=12207) 2024-12-04T20:28:10,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741970_1146 (size=4469) 2024-12-04T20:28:10,212 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#123 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:10,213 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/348d141c09bf4054b92efb73da51a8b2 is 175, key is test_row_0/A:col10/1733344089022/Put/seqid=0 2024-12-04T20:28:10,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047b903faf40454eb8b2870ebafc91f5e1_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344089138/Put/seqid=0 2024-12-04T20:28:10,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741971_1147 (size=31161) 2024-12-04T20:28:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:10,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:10,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741972_1148 (size=12254) 2024-12-04T20:28:10,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344150313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344150315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:10,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344150422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:10,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344150423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T20:28:10,614 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d955e344fec14f21acba3210015f4592 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d955e344fec14f21acba3210015f4592 2024-12-04T20:28:10,620 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into d955e344fec14f21acba3210015f4592(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:10,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:10,620 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=13, startTime=1733344090117; duration=0sec 2024-12-04T20:28:10,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:10,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:10,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:10,622 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:10,622 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:10,622 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:10,622 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/eec16a23481d4bf88b9a5e79c010b85c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/30d9dbb9896247208de8d7f10c1996d1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/82bc80acfb2c4fe3a74e76b6c181ceab] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=35.3 K 2024-12-04T20:28:10,622 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting eec16a23481d4bf88b9a5e79c010b85c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344085322 2024-12-04T20:28:10,623 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d9dbb9896247208de8d7f10c1996d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344087847 2024-12-04T20:28:10,623 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 82bc80acfb2c4fe3a74e76b6c181ceab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733344088456 2024-12-04T20:28:10,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344150631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:10,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344150642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:10,647 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:10,648 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/872abad0c68c4a73bc45ce9cbcc7d8ff is 50, key is test_row_0/C:col10/1733344089022/Put/seqid=0 2024-12-04T20:28:10,678 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/348d141c09bf4054b92efb73da51a8b2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/348d141c09bf4054b92efb73da51a8b2 2024-12-04T20:28:10,686 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into 348d141c09bf4054b92efb73da51a8b2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:10,686 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:10,686 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=13, startTime=1733344090116; duration=0sec 2024-12-04T20:28:10,686 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:10,686 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:10,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741973_1149 (size=12207) 2024-12-04T20:28:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:10,698 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047b903faf40454eb8b2870ebafc91f5e1_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047b903faf40454eb8b2870ebafc91f5e1_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/e1bc291aa35e46d4b31005f860efb18a, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/e1bc291aa35e46d4b31005f860efb18a is 175, key is test_row_0/A:col10/1733344089138/Put/seqid=0 2024-12-04T20:28:10,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741974_1150 (size=31055) 2024-12-04T20:28:10,743 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/e1bc291aa35e46d4b31005f860efb18a 2024-12-04T20:28:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/295c1d7dba814ee7b8cd33b66576824d is 50, key is test_row_0/B:col10/1733344089138/Put/seqid=0 2024-12-04T20:28:10,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741975_1151 (size=12101) 2024-12-04T20:28:10,786 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/295c1d7dba814ee7b8cd33b66576824d 2024-12-04T20:28:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/3bdba734bb1845a2856dfbae222bb5a4 is 50, key is test_row_0/C:col10/1733344089138/Put/seqid=0 2024-12-04T20:28:10,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741976_1152 (size=12101) 2024-12-04T20:28:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344150941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:10,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344150952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:11,097 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/872abad0c68c4a73bc45ce9cbcc7d8ff as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/872abad0c68c4a73bc45ce9cbcc7d8ff 2024-12-04T20:28:11,109 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into 872abad0c68c4a73bc45ce9cbcc7d8ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:11,109 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:11,109 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=13, startTime=1733344090117; duration=0sec 2024-12-04T20:28:11,109 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:11,109 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:11,232 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/3bdba734bb1845a2856dfbae222bb5a4 2024-12-04T20:28:11,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/e1bc291aa35e46d4b31005f860efb18a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/e1bc291aa35e46d4b31005f860efb18a 2024-12-04T20:28:11,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/e1bc291aa35e46d4b31005f860efb18a, entries=150, sequenceid=131, filesize=30.3 K 2024-12-04T20:28:11,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/295c1d7dba814ee7b8cd33b66576824d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/295c1d7dba814ee7b8cd33b66576824d 2024-12-04T20:28:11,251 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/295c1d7dba814ee7b8cd33b66576824d, entries=150, sequenceid=131, filesize=11.8 K 2024-12-04T20:28:11,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/3bdba734bb1845a2856dfbae222bb5a4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/3bdba734bb1845a2856dfbae222bb5a4 2024-12-04T20:28:11,258 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/3bdba734bb1845a2856dfbae222bb5a4, entries=150, sequenceid=131, filesize=11.8 K 2024-12-04T20:28:11,260 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1085ms, sequenceid=131, compaction requested=false 2024-12-04T20:28:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-04T20:28:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-04T20:28:11,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-04T20:28:11,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-12-04T20:28:11,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.8620 sec 2024-12-04T20:28:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:11,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T20:28:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:11,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ef4a3c5703d74ad38b8ae8be58d7bfaf_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344091452/Put/seqid=0 2024-12-04T20:28:11,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741977_1153 (size=14794) 2024-12-04T20:28:11,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:11,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344151489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:11,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344151492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:11,493 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ef4a3c5703d74ad38b8ae8be58d7bfaf_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ef4a3c5703d74ad38b8ae8be58d7bfaf_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:11,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/374583fb511842688819039bc4029f2a, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:11,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/374583fb511842688819039bc4029f2a is 175, key is test_row_0/A:col10/1733344091452/Put/seqid=0 2024-12-04T20:28:11,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741978_1154 (size=39749) 2024-12-04T20:28:11,544 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/374583fb511842688819039bc4029f2a 2024-12-04T20:28:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T20:28:11,552 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:11,553 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:11,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d6958e18a7aa4cf8b0979fc1b6871e57 is 50, key is test_row_0/B:col10/1733344091452/Put/seqid=0 2024-12-04T20:28:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-04T20:28:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T20:28:11,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:11,557 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:11,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:11,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741979_1155 (size=12151) 2024-12-04T20:28:11,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344151601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344151603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T20:28:11,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T20:28:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:11,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344151811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344151811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:11,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T20:28:11,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:11,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:11,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:11,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:11,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T20:28:11,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d6958e18a7aa4cf8b0979fc1b6871e57 2024-12-04T20:28:11,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6be61be02c0140eab4d953dc30d5f7d6 is 50, key is test_row_0/C:col10/1733344091452/Put/seqid=0 2024-12-04T20:28:12,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741980_1156 (size=12151) 2024-12-04T20:28:12,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T20:28:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344152121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344152121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:12,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T20:28:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T20:28:12,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T20:28:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:12,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6be61be02c0140eab4d953dc30d5f7d6 2024-12-04T20:28:12,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/374583fb511842688819039bc4029f2a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/374583fb511842688819039bc4029f2a 2024-12-04T20:28:12,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/374583fb511842688819039bc4029f2a, entries=200, sequenceid=148, filesize=38.8 K 2024-12-04T20:28:12,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/d6958e18a7aa4cf8b0979fc1b6871e57 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d6958e18a7aa4cf8b0979fc1b6871e57 2024-12-04T20:28:12,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d6958e18a7aa4cf8b0979fc1b6871e57, entries=150, sequenceid=148, filesize=11.9 K 2024-12-04T20:28:12,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6be61be02c0140eab4d953dc30d5f7d6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6be61be02c0140eab4d953dc30d5f7d6 2024-12-04T20:28:12,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6be61be02c0140eab4d953dc30d5f7d6, entries=150, sequenceid=148, filesize=11.9 K 2024-12-04T20:28:12,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 987ms, sequenceid=148, compaction requested=true 2024-12-04T20:28:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:12,440 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:12,440 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:12,442 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:12,442 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:12,442 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,442 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/348d141c09bf4054b92efb73da51a8b2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/e1bc291aa35e46d4b31005f860efb18a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/374583fb511842688819039bc4029f2a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=99.6 K 2024-12-04T20:28:12,442 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,442 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/348d141c09bf4054b92efb73da51a8b2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/e1bc291aa35e46d4b31005f860efb18a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/374583fb511842688819039bc4029f2a] 2024-12-04T20:28:12,443 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:12,443 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:12,443 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,443 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d955e344fec14f21acba3210015f4592, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/295c1d7dba814ee7b8cd33b66576824d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d6958e18a7aa4cf8b0979fc1b6871e57] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=35.6 K 2024-12-04T20:28:12,444 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d955e344fec14f21acba3210015f4592, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733344088456 2024-12-04T20:28:12,444 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 348d141c09bf4054b92efb73da51a8b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733344088456 2024-12-04T20:28:12,446 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 295c1d7dba814ee7b8cd33b66576824d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733344089129 2024-12-04T20:28:12,447 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1bc291aa35e46d4b31005f860efb18a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733344089129 2024-12-04T20:28:12,447 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 374583fb511842688819039bc4029f2a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733344090295 2024-12-04T20:28:12,447 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d6958e18a7aa4cf8b0979fc1b6871e57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733344090295 2024-12-04T20:28:12,462 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:12,471 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#133 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:12,472 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/2aab068641b549d9b29349f2482ce06a is 50, key is test_row_0/B:col10/1733344091452/Put/seqid=0 2024-12-04T20:28:12,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T20:28:12,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T20:28:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:12,490 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204dee7c55caa254a68ac91b3467553d6e5_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:12,492 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204dee7c55caa254a68ac91b3467553d6e5_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:12,492 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dee7c55caa254a68ac91b3467553d6e5_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:12,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741981_1157 (size=12459) 2024-12-04T20:28:12,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741982_1158 (size=4469) 2024-12-04T20:28:12,553 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#132 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:12,553 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/6658cf217321484ea0af8400f55237a9 is 175, key is test_row_0/A:col10/1733344091452/Put/seqid=0 2024-12-04T20:28:12,555 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/2aab068641b549d9b29349f2482ce06a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2aab068641b549d9b29349f2482ce06a 2024-12-04T20:28:12,562 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into 2aab068641b549d9b29349f2482ce06a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:12,562 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:12,562 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=13, startTime=1733344092440; duration=0sec 2024-12-04T20:28:12,563 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:12,563 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:12,563 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:12,564 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:12,564 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:12,564 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:12,564 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/872abad0c68c4a73bc45ce9cbcc7d8ff, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/3bdba734bb1845a2856dfbae222bb5a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6be61be02c0140eab4d953dc30d5f7d6] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=35.6 K 2024-12-04T20:28:12,566 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 872abad0c68c4a73bc45ce9cbcc7d8ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733344088456 2024-12-04T20:28:12,567 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bdba734bb1845a2856dfbae222bb5a4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733344089129 2024-12-04T20:28:12,567 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be61be02c0140eab4d953dc30d5f7d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733344090295 2024-12-04T20:28:12,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041a290c24380840d1b6bd9e5f83caba8a_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344091478/Put/seqid=0 2024-12-04T20:28:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741983_1159 (size=31413) 2024-12-04T20:28:12,583 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:12,584 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/887b8472e69c42bf867e4d5063e58ee5 is 50, key is test_row_0/C:col10/1733344091452/Put/seqid=0 2024-12-04T20:28:12,592 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/6658cf217321484ea0af8400f55237a9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6658cf217321484ea0af8400f55237a9 2024-12-04T20:28:12,598 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into 6658cf217321484ea0af8400f55237a9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:12,598 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:12,598 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=13, startTime=1733344092440; duration=0sec 2024-12-04T20:28:12,598 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:12,598 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:12,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:12,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741984_1160 (size=12304) 2024-12-04T20:28:12,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741985_1161 (size=12459) 2024-12-04T20:28:12,655 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/887b8472e69c42bf867e4d5063e58ee5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/887b8472e69c42bf867e4d5063e58ee5 2024-12-04T20:28:12,661 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into 887b8472e69c42bf867e4d5063e58ee5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:12,661 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:12,661 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=13, startTime=1733344092441; duration=0sec 2024-12-04T20:28:12,662 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:12,662 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:12,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344152663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344152665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T20:28:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344152771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344152771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:12,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344152982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344152982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:13,040 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041a290c24380840d1b6bd9e5f83caba8a_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041a290c24380840d1b6bd9e5f83caba8a_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:13,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/aaec981d9b32472dbde35d9b18c218c2, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:13,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/aaec981d9b32472dbde35d9b18c218c2 is 175, key is test_row_0/A:col10/1733344091478/Put/seqid=0 2024-12-04T20:28:13,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741986_1162 (size=31105) 2024-12-04T20:28:13,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/aaec981d9b32472dbde35d9b18c218c2 2024-12-04T20:28:13,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/154db25c19e84d30baf48e01f7a9a09b is 50, key is test_row_0/B:col10/1733344091478/Put/seqid=0 2024-12-04T20:28:13,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741987_1163 (size=12151) 2024-12-04T20:28:13,089 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/154db25c19e84d30baf48e01f7a9a09b 2024-12-04T20:28:13,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/87aa56fbb9a64f43ae2a3e696caa31d7 is 50, key is test_row_0/C:col10/1733344091478/Put/seqid=0 2024-12-04T20:28:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741988_1164 (size=12151) 2024-12-04T20:28:13,148 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/87aa56fbb9a64f43ae2a3e696caa31d7 2024-12-04T20:28:13,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/aaec981d9b32472dbde35d9b18c218c2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/aaec981d9b32472dbde35d9b18c218c2 2024-12-04T20:28:13,165 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/aaec981d9b32472dbde35d9b18c218c2, entries=150, sequenceid=166, filesize=30.4 K 2024-12-04T20:28:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/154db25c19e84d30baf48e01f7a9a09b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/154db25c19e84d30baf48e01f7a9a09b 2024-12-04T20:28:13,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/154db25c19e84d30baf48e01f7a9a09b, entries=150, sequenceid=166, filesize=11.9 K 2024-12-04T20:28:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/87aa56fbb9a64f43ae2a3e696caa31d7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/87aa56fbb9a64f43ae2a3e696caa31d7 2024-12-04T20:28:13,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/87aa56fbb9a64f43ae2a3e696caa31d7, entries=150, sequenceid=166, filesize=11.9 K 2024-12-04T20:28:13,187 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 707ms, sequenceid=166, compaction requested=false 2024-12-04T20:28:13,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:13,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:13,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-04T20:28:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-04T20:28:13,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-04T20:28:13,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-12-04T20:28:13,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6420 sec 2024-12-04T20:28:13,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T20:28:13,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:13,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:13,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:13,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:13,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:13,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:13,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a523ba1638e24d198ef65182cd015c4e_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344093294/Put/seqid=0 2024-12-04T20:28:13,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741989_1165 (size=17284) 2024-12-04T20:28:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344153339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344153337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344153451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344153451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344153542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,544 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344153582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344153582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,585 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,586 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344153661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344153661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T20:28:13,701 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:13,702 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-04T20:28:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T20:28:13,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:13,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:13,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:13,737 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:13,742 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a523ba1638e24d198ef65182cd015c4e_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a523ba1638e24d198ef65182cd015c4e_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:13,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c68e73266f6343f5b49f15c8bc6cbb28, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:13,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c68e73266f6343f5b49f15c8bc6cbb28 is 175, key is test_row_0/A:col10/1733344093294/Put/seqid=0 2024-12-04T20:28:13,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741990_1166 (size=48389) 2024-12-04T20:28:13,748 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c68e73266f6343f5b49f15c8bc6cbb28 2024-12-04T20:28:13,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7849f3cf1b6549f994299e71507e6f6d is 50, key is test_row_0/B:col10/1733344093294/Put/seqid=0 2024-12-04T20:28:13,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741991_1167 (size=12151) 2024-12-04T20:28:13,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T20:28:13,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T20:28:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:13,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344153971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344153971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:14,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T20:28:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T20:28:14,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T20:28:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:14,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7849f3cf1b6549f994299e71507e6f6d 2024-12-04T20:28:14,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/363b9145e38a4486bcac73e72c4effa0 is 50, key is test_row_0/C:col10/1733344093294/Put/seqid=0 2024-12-04T20:28:14,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741992_1168 (size=12151) 2024-12-04T20:28:14,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/363b9145e38a4486bcac73e72c4effa0 2024-12-04T20:28:14,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/c68e73266f6343f5b49f15c8bc6cbb28 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c68e73266f6343f5b49f15c8bc6cbb28 2024-12-04T20:28:14,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c68e73266f6343f5b49f15c8bc6cbb28, entries=250, sequenceid=185, filesize=47.3 K 2024-12-04T20:28:14,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7849f3cf1b6549f994299e71507e6f6d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7849f3cf1b6549f994299e71507e6f6d 2024-12-04T20:28:14,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7849f3cf1b6549f994299e71507e6f6d, entries=150, sequenceid=185, filesize=11.9 K 2024-12-04T20:28:14,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/363b9145e38a4486bcac73e72c4effa0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/363b9145e38a4486bcac73e72c4effa0 2024-12-04T20:28:14,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/363b9145e38a4486bcac73e72c4effa0, entries=150, sequenceid=185, filesize=11.9 K 2024-12-04T20:28:14,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 926ms, sequenceid=185, compaction requested=true 2024-12-04T20:28:14,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:14,224 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:14,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:14,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:14,225 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:14,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:14,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:14,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:14,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:14,226 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:14,226 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:14,226 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,226 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2aab068641b549d9b29349f2482ce06a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/154db25c19e84d30baf48e01f7a9a09b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7849f3cf1b6549f994299e71507e6f6d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=35.9 K 2024-12-04T20:28:14,226 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:14,227 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:14,227 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,227 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6658cf217321484ea0af8400f55237a9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/aaec981d9b32472dbde35d9b18c218c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c68e73266f6343f5b49f15c8bc6cbb28] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=108.3 K 2024-12-04T20:28:14,227 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,227 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6658cf217321484ea0af8400f55237a9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/aaec981d9b32472dbde35d9b18c218c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c68e73266f6343f5b49f15c8bc6cbb28] 2024-12-04T20:28:14,227 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aab068641b549d9b29349f2482ce06a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733344090295 2024-12-04T20:28:14,227 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6658cf217321484ea0af8400f55237a9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733344090295 2024-12-04T20:28:14,227 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 154db25c19e84d30baf48e01f7a9a09b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733344091478 2024-12-04T20:28:14,228 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaec981d9b32472dbde35d9b18c218c2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733344091478 2024-12-04T20:28:14,228 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7849f3cf1b6549f994299e71507e6f6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733344092660 2024-12-04T20:28:14,228 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c68e73266f6343f5b49f15c8bc6cbb28, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733344092655 2024-12-04T20:28:14,251 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:14,252 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3f0ce740941c4f1792e05e0c6dbb2e7f is 50, key is test_row_0/B:col10/1733344093294/Put/seqid=0 2024-12-04T20:28:14,266 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:14,290 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204f301d578060d4d05992e4ca58d2b1256_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:14,291 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204f301d578060d4d05992e4ca58d2b1256_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:14,292 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f301d578060d4d05992e4ca58d2b1256_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:14,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741993_1169 (size=12561) 2024-12-04T20:28:14,307 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3f0ce740941c4f1792e05e0c6dbb2e7f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f0ce740941c4f1792e05e0c6dbb2e7f 2024-12-04T20:28:14,315 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into 3f0ce740941c4f1792e05e0c6dbb2e7f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:14,315 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:14,315 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=13, startTime=1733344094225; duration=0sec 2024-12-04T20:28:14,315 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:14,316 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:14,316 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:14,320 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:14,320 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:14,320 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,320 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/887b8472e69c42bf867e4d5063e58ee5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/87aa56fbb9a64f43ae2a3e696caa31d7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/363b9145e38a4486bcac73e72c4effa0] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=35.9 K 2024-12-04T20:28:14,321 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 887b8472e69c42bf867e4d5063e58ee5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733344090295 2024-12-04T20:28:14,321 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 87aa56fbb9a64f43ae2a3e696caa31d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733344091478 2024-12-04T20:28:14,322 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 363b9145e38a4486bcac73e72c4effa0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733344092660 2024-12-04T20:28:14,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T20:28:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:14,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T20:28:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T20:28:14,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741994_1170 (size=4469) 2024-12-04T20:28:14,343 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#142 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:14,343 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/a28edbbf2feb4d68880dc7af25e0003f is 175, key is test_row_0/A:col10/1733344093294/Put/seqid=0 2024-12-04T20:28:14,351 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:14,352 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/8325384bc1604991908f101513c1873d is 50, key is test_row_0/C:col10/1733344093294/Put/seqid=0 2024-12-04T20:28:14,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047be1cdb0ae54498e8a89537ddfdbd403_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344093328/Put/seqid=0 2024-12-04T20:28:14,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741996_1172 (size=12561) 2024-12-04T20:28:14,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741995_1171 (size=31515) 2024-12-04T20:28:14,414 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/a28edbbf2feb4d68880dc7af25e0003f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/a28edbbf2feb4d68880dc7af25e0003f 2024-12-04T20:28:14,421 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into a28edbbf2feb4d68880dc7af25e0003f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:14,422 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:14,422 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=13, startTime=1733344094224; duration=0sec 2024-12-04T20:28:14,422 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:14,422 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:14,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741997_1173 (size=12304) 2024-12-04T20:28:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:14,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344154504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:14,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344154509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344154612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344154622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:14,793 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/8325384bc1604991908f101513c1873d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8325384bc1604991908f101513c1873d 2024-12-04T20:28:14,797 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into 8325384bc1604991908f101513c1873d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:14,797 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:14,797 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=13, startTime=1733344094225; duration=0sec 2024-12-04T20:28:14,798 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:14,798 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:14,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344154821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:14,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:14,831 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047be1cdb0ae54498e8a89537ddfdbd403_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047be1cdb0ae54498e8a89537ddfdbd403_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:14,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/71c713b6186c48f9a879fba2faec5f75, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:14,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/71c713b6186c48f9a879fba2faec5f75 is 175, key is test_row_0/A:col10/1733344093328/Put/seqid=0 2024-12-04T20:28:14,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344154832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:14,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741998_1174 (size=31105) 2024-12-04T20:28:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T20:28:15,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344155132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:15,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344155141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:15,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/71c713b6186c48f9a879fba2faec5f75 2024-12-04T20:28:15,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7772ed14aa474e6c83677d3985edb21c is 50, key is test_row_0/B:col10/1733344093328/Put/seqid=0 2024-12-04T20:28:15,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741999_1175 (size=12151) 2024-12-04T20:28:15,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344155651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344155651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:15,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7772ed14aa474e6c83677d3985edb21c 2024-12-04T20:28:15,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/80d2a421c22f41d0a3bf93e73b6dca81 is 50, key is test_row_0/C:col10/1733344093328/Put/seqid=0 2024-12-04T20:28:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742000_1176 (size=12151) 2024-12-04T20:28:15,690 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/80d2a421c22f41d0a3bf93e73b6dca81 2024-12-04T20:28:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/71c713b6186c48f9a879fba2faec5f75 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/71c713b6186c48f9a879fba2faec5f75 2024-12-04T20:28:15,706 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/71c713b6186c48f9a879fba2faec5f75, entries=150, sequenceid=202, filesize=30.4 K 2024-12-04T20:28:15,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7772ed14aa474e6c83677d3985edb21c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7772ed14aa474e6c83677d3985edb21c 2024-12-04T20:28:15,712 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7772ed14aa474e6c83677d3985edb21c, entries=150, sequenceid=202, filesize=11.9 K 2024-12-04T20:28:15,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/80d2a421c22f41d0a3bf93e73b6dca81 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/80d2a421c22f41d0a3bf93e73b6dca81 2024-12-04T20:28:15,717 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/80d2a421c22f41d0a3bf93e73b6dca81, entries=150, sequenceid=202, filesize=11.9 K 2024-12-04T20:28:15,718 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1395ms, sequenceid=202, compaction requested=false 2024-12-04T20:28:15,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:15,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:15,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-04T20:28:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-04T20:28:15,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-04T20:28:15,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-12-04T20:28:15,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.0200 sec 2024-12-04T20:28:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T20:28:15,852 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:15,852 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-04T20:28:15,854 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T20:28:15,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:15,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T20:28:16,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T20:28:16,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:16,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T20:28:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204757c89101a8848c8a4fa243f5b2e3d08_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344094508/Put/seqid=0 2024-12-04T20:28:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742001_1177 (size=12304) 2024-12-04T20:28:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204757c89101a8848c8a4fa243f5b2e3d08_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204757c89101a8848c8a4fa243f5b2e3d08_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:16,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/483ba75d41f046c097c666fe9335a1a4, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:16,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/483ba75d41f046c097c666fe9335a1a4 is 175, key is test_row_0/A:col10/1733344094508/Put/seqid=0 2024-12-04T20:28:16,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742002_1178 (size=31105) 2024-12-04T20:28:16,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/483ba75d41f046c097c666fe9335a1a4 2024-12-04T20:28:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/2876738038684202b58317e42ba35bb1 is 50, key is test_row_0/B:col10/1733344094508/Put/seqid=0 2024-12-04T20:28:16,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742003_1179 (size=12151) 2024-12-04T20:28:16,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/2876738038684202b58317e42ba35bb1 2024-12-04T20:28:16,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6ee536f193c941b58600ba8110cd554a is 50, key is test_row_0/C:col10/1733344094508/Put/seqid=0 2024-12-04T20:28:16,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742004_1180 (size=12151) 2024-12-04T20:28:16,120 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6ee536f193c941b58600ba8110cd554a 2024-12-04T20:28:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/483ba75d41f046c097c666fe9335a1a4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/483ba75d41f046c097c666fe9335a1a4 2024-12-04T20:28:16,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/483ba75d41f046c097c666fe9335a1a4, entries=150, sequenceid=220, filesize=30.4 K 2024-12-04T20:28:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/2876738038684202b58317e42ba35bb1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2876738038684202b58317e42ba35bb1 2024-12-04T20:28:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,162 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2876738038684202b58317e42ba35bb1, entries=150, sequenceid=220, filesize=11.9 K 2024-12-04T20:28:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6ee536f193c941b58600ba8110cd554a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6ee536f193c941b58600ba8110cd554a 2024-12-04T20:28:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,169 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6ee536f193c941b58600ba8110cd554a, entries=150, sequenceid=220, filesize=11.9 K 2024-12-04T20:28:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,170 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 162ms, sequenceid=220, compaction requested=true 2024-12-04T20:28:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-04T20:28:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-04T20:28:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T20:28:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-04T20:28:16,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 317 msec 2024-12-04T20:28:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 322 msec 2024-12-04T20:28:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T20:28:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,482 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,482 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-04T20:28:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T20:28:16,484 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,485 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:16,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T20:28:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T20:28:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-04T20:28:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-04T20:28:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-04T20:28:16,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-04T20:28:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 161 msec 2024-12-04T20:28:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:28:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:16,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:16,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:16,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:16,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:16,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120406b4efe9311e4695b68468cef40263b5_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344096722/Put/seqid=0 2024-12-04T20:28:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T20:28:16,802 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:16,803 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-04T20:28:16,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T20:28:16,805 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:16,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:16,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742006_1182 (size=24758) 2024-12-04T20:28:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344156829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:16,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344096799, max timestamp=1733344096836Num uniques 859; 2024-12-04T20:28:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:16,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:28:16,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344096820, max timestamp=1733344096836Num uniques 859; 2024-12-04T20:28:16,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:16,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344096820, max timestamp=1733344096836Num uniques 859; 2024-12-04T20:28:16,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:16,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344156902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T20:28:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344156941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:16,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T20:28:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:16,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344157011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:17,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T20:28:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T20:28:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344157151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:17,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344157222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:17,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:17,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120406b4efe9311e4695b68468cef40263b5_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120406b4efe9311e4695b68468cef40263b5_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:17,233 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/524be53d7f2249af9302a2f4451979a4, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:17,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/524be53d7f2249af9302a2f4451979a4 is 175, key is test_row_0/A:col10/1733344096722/Put/seqid=0 2024-12-04T20:28:17,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742005_1181 (size=74395) 2024-12-04T20:28:17,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T20:28:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T20:28:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T20:28:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344157461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:17,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344157531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:17,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T20:28:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/524be53d7f2249af9302a2f4451979a4 2024-12-04T20:28:17,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/30c77e45e57146af85785f21982f26da is 50, key is test_row_0/B:col10/1733344096722/Put/seqid=0 2024-12-04T20:28:17,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742007_1183 (size=12151) 2024-12-04T20:28:17,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/30c77e45e57146af85785f21982f26da 2024-12-04T20:28:17,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/32b96d0350284aad9abcdbbe3bf585ec is 50, key is test_row_0/C:col10/1733344096722/Put/seqid=0 2024-12-04T20:28:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742008_1184 (size=12151) 2024-12-04T20:28:17,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/32b96d0350284aad9abcdbbe3bf585ec 2024-12-04T20:28:17,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/524be53d7f2249af9302a2f4451979a4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/524be53d7f2249af9302a2f4451979a4 2024-12-04T20:28:17,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/524be53d7f2249af9302a2f4451979a4, entries=400, sequenceid=232, filesize=72.7 K 2024-12-04T20:28:17,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/30c77e45e57146af85785f21982f26da as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/30c77e45e57146af85785f21982f26da 2024-12-04T20:28:17,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T20:28:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:17,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/30c77e45e57146af85785f21982f26da, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T20:28:17,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/32b96d0350284aad9abcdbbe3bf585ec as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/32b96d0350284aad9abcdbbe3bf585ec 2024-12-04T20:28:17,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/32b96d0350284aad9abcdbbe3bf585ec, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T20:28:17,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1026ms, sequenceid=232, compaction requested=true 2024-12-04T20:28:17,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:17,749 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:17,751 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 168120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:17,751 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:17,751 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,751 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/a28edbbf2feb4d68880dc7af25e0003f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/71c713b6186c48f9a879fba2faec5f75, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/483ba75d41f046c097c666fe9335a1a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/524be53d7f2249af9302a2f4451979a4] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=164.2 K 2024-12-04T20:28:17,751 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,751 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/a28edbbf2feb4d68880dc7af25e0003f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/71c713b6186c48f9a879fba2faec5f75, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/483ba75d41f046c097c666fe9335a1a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/524be53d7f2249af9302a2f4451979a4] 2024-12-04T20:28:17,752 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28edbbf2feb4d68880dc7af25e0003f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733344092660 2024-12-04T20:28:17,752 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c713b6186c48f9a879fba2faec5f75, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733344093320 2024-12-04T20:28:17,753 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483ba75d41f046c097c666fe9335a1a4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733344094501 2024-12-04T20:28:17,753 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524be53d7f2249af9302a2f4451979a4, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733344096693 2024-12-04T20:28:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:17,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:17,765 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:17,766 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:17,766 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:17,767 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,767 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f0ce740941c4f1792e05e0c6dbb2e7f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7772ed14aa474e6c83677d3985edb21c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2876738038684202b58317e42ba35bb1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/30c77e45e57146af85785f21982f26da] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=47.9 K 2024-12-04T20:28:17,767 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0ce740941c4f1792e05e0c6dbb2e7f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733344092660 2024-12-04T20:28:17,767 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7772ed14aa474e6c83677d3985edb21c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733344093320 2024-12-04T20:28:17,768 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2876738038684202b58317e42ba35bb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733344094501 2024-12-04T20:28:17,768 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c77e45e57146af85785f21982f26da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733344096709 2024-12-04T20:28:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:17,778 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:17,788 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#154 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:17,789 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/b98cca193f754e88a1af931a3286f9fa is 50, key is test_row_0/B:col10/1733344096722/Put/seqid=0 2024-12-04T20:28:17,809 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412040636af67392b4dd8a71a9019e21d7950_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:17,812 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412040636af67392b4dd8a71a9019e21d7950_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:17,812 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040636af67392b4dd8a71a9019e21d7950_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742009_1185 (size=12697) 2024-12-04T20:28:17,847 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/b98cca193f754e88a1af931a3286f9fa as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b98cca193f754e88a1af931a3286f9fa 2024-12-04T20:28:17,855 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into b98cca193f754e88a1af931a3286f9fa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:17,855 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:17,855 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=12, startTime=1733344097765; duration=0sec 2024-12-04T20:28:17,855 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:17,855 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:17,855 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:17,857 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:17,857 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:17,857 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,857 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8325384bc1604991908f101513c1873d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/80d2a421c22f41d0a3bf93e73b6dca81, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6ee536f193c941b58600ba8110cd554a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/32b96d0350284aad9abcdbbe3bf585ec] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=47.9 K 2024-12-04T20:28:17,858 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8325384bc1604991908f101513c1873d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733344092660 2024-12-04T20:28:17,858 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d2a421c22f41d0a3bf93e73b6dca81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733344093320 2024-12-04T20:28:17,859 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee536f193c941b58600ba8110cd554a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733344094501 2024-12-04T20:28:17,859 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 32b96d0350284aad9abcdbbe3bf585ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733344096709 2024-12-04T20:28:17,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742010_1186 (size=4469) 2024-12-04T20:28:17,868 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#153 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:17,869 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/f1a75af4a1eb4dc1bb790057648136fd is 175, key is test_row_0/A:col10/1733344096722/Put/seqid=0 2024-12-04T20:28:17,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T20:28:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:17,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-12-04T20:28:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:17,890 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#155 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:17,891 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/42f971a0376544c282cb16628bafd8d1 is 50, key is test_row_0/C:col10/1733344096722/Put/seqid=0 2024-12-04T20:28:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T20:28:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742011_1187 (size=31651) 2024-12-04T20:28:17,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045f3d47f0020748a7856035b0097942f0_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344096894/Put/seqid=0 2024-12-04T20:28:17,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:17,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742012_1188 (size=12697) 2024-12-04T20:28:18,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742013_1189 (size=12454) 2024-12-04T20:28:18,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,025 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045f3d47f0020748a7856035b0097942f0_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045f3d47f0020748a7856035b0097942f0_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:18,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/773f6e9e96ac4ceca3ddce83a1f1f5fd, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/773f6e9e96ac4ceca3ddce83a1f1f5fd is 175, key is test_row_0/A:col10/1733344096894/Put/seqid=0 2024-12-04T20:28:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344158037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344158042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:18,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742014_1190 (size=31255) 2024-12-04T20:28:18,076 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/773f6e9e96ac4ceca3ddce83a1f1f5fd 2024-12-04T20:28:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e is 50, key is test_row_0/B:col10/1733344096894/Put/seqid=0 2024-12-04T20:28:18,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742015_1191 (size=12301) 2024-12-04T20:28:18,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e 2024-12-04T20:28:18,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/69f077dcdda54092a13130448b150394 is 50, key is test_row_0/C:col10/1733344096894/Put/seqid=0 2024-12-04T20:28:18,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344158151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:18,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742016_1192 (size=12301) 2024-12-04T20:28:18,183 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/69f077dcdda54092a13130448b150394 2024-12-04T20:28:18,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/773f6e9e96ac4ceca3ddce83a1f1f5fd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/773f6e9e96ac4ceca3ddce83a1f1f5fd 2024-12-04T20:28:18,195 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/773f6e9e96ac4ceca3ddce83a1f1f5fd, entries=150, sequenceid=268, filesize=30.5 K 2024-12-04T20:28:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e 2024-12-04T20:28:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,210 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e, entries=150, sequenceid=268, filesize=12.0 K 2024-12-04T20:28:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/69f077dcdda54092a13130448b150394 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/69f077dcdda54092a13130448b150394 2024-12-04T20:28:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,220 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/69f077dcdda54092a13130448b150394, entries=150, sequenceid=268, filesize=12.0 K 2024-12-04T20:28:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 357ms, sequenceid=268, compaction requested=false 2024-12-04T20:28:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-04T20:28:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-04T20:28:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-04T20:28:18,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-12-04T20:28:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4390 sec 2024-12-04T20:28:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,289 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T20:28:18,289 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T20:28:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,354 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/f1a75af4a1eb4dc1bb790057648136fd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/f1a75af4a1eb4dc1bb790057648136fd 2024-12-04T20:28:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,360 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into f1a75af4a1eb4dc1bb790057648136fd(size=30.9 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:18,360 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:18,360 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=12, startTime=1733344097749; duration=0sec 2024-12-04T20:28:18,360 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:18,360 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:28:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204261f01f54bb44acdb9c9d9b1310f8a12_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344098034/Put/seqid=0 2024-12-04T20:28:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,396 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/42f971a0376544c282cb16628bafd8d1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/42f971a0376544c282cb16628bafd8d1 2024-12-04T20:28:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,404 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into 42f971a0376544c282cb16628bafd8d1(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:18,404 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:18,404 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=12, startTime=1733344097776; duration=0sec 2024-12-04T20:28:18,404 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:18,404 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742017_1193 (size=12454) 2024-12-04T20:28:18,431 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:18,439 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204261f01f54bb44acdb9c9d9b1310f8a12_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204261f01f54bb44acdb9c9d9b1310f8a12_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:18,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/673f49c1b40942b585cb7f369c591210, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:18,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/673f49c1b40942b585cb7f369c591210 is 175, key is test_row_0/A:col10/1733344098034/Put/seqid=0 2024-12-04T20:28:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742018_1194 (size=31251) 2024-12-04T20:28:18,492 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/673f49c1b40942b585cb7f369c591210 2024-12-04T20:28:18,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/de5e9b07fe244decaff0686242064085 is 50, key is test_row_0/B:col10/1733344098034/Put/seqid=0 2024-12-04T20:28:18,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344158521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:18,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:18,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742019_1195 (size=9857) 2024-12-04T20:28:18,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/de5e9b07fe244decaff0686242064085 2024-12-04T20:28:18,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/73decd68acbe4acc9a8e04339e9f658d is 50, key is test_row_0/C:col10/1733344098034/Put/seqid=0 2024-12-04T20:28:18,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742020_1196 (size=9857) 2024-12-04T20:28:18,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/73decd68acbe4acc9a8e04339e9f658d 2024-12-04T20:28:18,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/673f49c1b40942b585cb7f369c591210 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/673f49c1b40942b585cb7f369c591210 2024-12-04T20:28:18,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/673f49c1b40942b585cb7f369c591210, entries=150, sequenceid=281, filesize=30.5 K 2024-12-04T20:28:18,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/de5e9b07fe244decaff0686242064085 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/de5e9b07fe244decaff0686242064085 2024-12-04T20:28:18,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/de5e9b07fe244decaff0686242064085, entries=100, sequenceid=281, filesize=9.6 K 2024-12-04T20:28:18,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/73decd68acbe4acc9a8e04339e9f658d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/73decd68acbe4acc9a8e04339e9f658d 2024-12-04T20:28:18,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/73decd68acbe4acc9a8e04339e9f658d, entries=100, sequenceid=281, filesize=9.6 K 2024-12-04T20:28:18,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 237ms, sequenceid=281, compaction requested=true 2024-12-04T20:28:18,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:18,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:18,612 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94157 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:18,612 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:18,612 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:18,612 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/f1a75af4a1eb4dc1bb790057648136fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/773f6e9e96ac4ceca3ddce83a1f1f5fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/673f49c1b40942b585cb7f369c591210] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=92.0 K 2024-12-04T20:28:18,612 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:18,612 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/f1a75af4a1eb4dc1bb790057648136fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/773f6e9e96ac4ceca3ddce83a1f1f5fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/673f49c1b40942b585cb7f369c591210] 2024-12-04T20:28:18,613 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a75af4a1eb4dc1bb790057648136fd, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733344096709 2024-12-04T20:28:18,613 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 773f6e9e96ac4ceca3ddce83a1f1f5fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733344096891 2024-12-04T20:28:18,614 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673f49c1b40942b585cb7f369c591210, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733344098021 2024-12-04T20:28:18,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:18,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:18,619 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:18,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:18,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:18,620 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:18,620 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b98cca193f754e88a1af931a3286f9fa, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/de5e9b07fe244decaff0686242064085] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=34.0 K 2024-12-04T20:28:18,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b98cca193f754e88a1af931a3286f9fa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733344096709 2024-12-04T20:28:18,621 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf6b5e16e3b49a89d37c0d37b2a1b3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733344096891 2024-12-04T20:28:18,621 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting de5e9b07fe244decaff0686242064085, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733344098034 2024-12-04T20:28:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:18,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:18,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:18,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:18,629 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:18,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T20:28:18,635 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:18,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:18,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:18,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:18,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:18,636 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/808706606ed8494cabdcf4841fc08785 is 50, key is test_row_0/B:col10/1733344098034/Put/seqid=0 2024-12-04T20:28:18,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:18,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:18,641 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412042aeb3a45a9364c06a06b1948eee8ba30_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:18,643 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412042aeb3a45a9364c06a06b1948eee8ba30_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:18,643 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042aeb3a45a9364c06a06b1948eee8ba30_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:18,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742022_1198 (size=4469) 2024-12-04T20:28:18,675 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#162 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:18,675 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/553798a837054c0dbab00c4e5d9457d9 is 175, key is test_row_0/A:col10/1733344098034/Put/seqid=0 2024-12-04T20:28:18,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204880d752703af43e5aceab0ce37c66c2e_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344098518/Put/seqid=0 2024-12-04T20:28:18,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344158686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:18,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742021_1197 (size=12949) 2024-12-04T20:28:18,722 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/808706606ed8494cabdcf4841fc08785 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/808706606ed8494cabdcf4841fc08785 2024-12-04T20:28:18,732 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into 808706606ed8494cabdcf4841fc08785(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:18,732 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:18,732 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=13, startTime=1733344098619; duration=0sec 2024-12-04T20:28:18,732 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:18,732 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:18,732 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:18,735 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:18,735 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:18,735 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:18,736 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/42f971a0376544c282cb16628bafd8d1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/69f077dcdda54092a13130448b150394, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/73decd68acbe4acc9a8e04339e9f658d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=34.0 K 2024-12-04T20:28:18,737 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f971a0376544c282cb16628bafd8d1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733344096709 2024-12-04T20:28:18,738 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f077dcdda54092a13130448b150394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733344096891 2024-12-04T20:28:18,738 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 73decd68acbe4acc9a8e04339e9f658d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733344098034 2024-12-04T20:28:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742023_1199 (size=32010) 2024-12-04T20:28:18,767 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/553798a837054c0dbab00c4e5d9457d9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/553798a837054c0dbab00c4e5d9457d9 2024-12-04T20:28:18,773 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into 553798a837054c0dbab00c4e5d9457d9(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:18,773 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:18,773 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=13, startTime=1733344098610; duration=0sec 2024-12-04T20:28:18,773 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:18,773 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:18,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742024_1200 (size=12454) 2024-12-04T20:28:18,787 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:18,788 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6bf3f086929c48e880dca2753c0062d2 is 50, key is test_row_0/C:col10/1733344098034/Put/seqid=0 2024-12-04T20:28:18,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344158801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:18,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742025_1201 (size=12949) 2024-12-04T20:28:18,829 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/6bf3f086929c48e880dca2753c0062d2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6bf3f086929c48e880dca2753c0062d2 2024-12-04T20:28:18,834 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into 6bf3f086929c48e880dca2753c0062d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:18,834 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:18,834 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=13, startTime=1733344098623; duration=0sec 2024-12-04T20:28:18,834 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:18,834 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T20:28:18,962 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:18,962 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-04T20:28:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T20:28:18,965 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:18,965 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:18,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:19,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344159011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:19,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344159062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:19,064 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T20:28:19,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:19,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204880d752703af43e5aceab0ce37c66c2e_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204880d752703af43e5aceab0ce37c66c2e_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:19,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/55861e0c123b4698959394ebdb0d9269, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:19,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/55861e0c123b4698959394ebdb0d9269 is 175, key is test_row_0/A:col10/1733344098518/Put/seqid=0 2024-12-04T20:28:19,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742026_1202 (size=31255) 2024-12-04T20:28:19,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T20:28:19,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344159321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:19,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,598 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/55861e0c123b4698959394ebdb0d9269 2024-12-04T20:28:19,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/166a5b755fab413bbfe17fd83068deee is 50, key is test_row_0/B:col10/1733344098518/Put/seqid=0 2024-12-04T20:28:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T20:28:19,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742027_1203 (size=12301) 2024-12-04T20:28:19,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:19,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344159831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:19,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:19,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:20,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:20,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:20,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/166a5b755fab413bbfe17fd83068deee 2024-12-04T20:28:20,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/57591d96c38f410f9cae101c9c724a3e is 50, key is test_row_0/C:col10/1733344098518/Put/seqid=0 2024-12-04T20:28:20,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742028_1204 (size=12301) 2024-12-04T20:28:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T20:28:20,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/57591d96c38f410f9cae101c9c724a3e 2024-12-04T20:28:20,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/55861e0c123b4698959394ebdb0d9269 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/55861e0c123b4698959394ebdb0d9269 2024-12-04T20:28:20,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/55861e0c123b4698959394ebdb0d9269, entries=150, sequenceid=303, filesize=30.5 K 2024-12-04T20:28:20,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/166a5b755fab413bbfe17fd83068deee as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/166a5b755fab413bbfe17fd83068deee 2024-12-04T20:28:20,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/166a5b755fab413bbfe17fd83068deee, entries=150, sequenceid=303, filesize=12.0 K 2024-12-04T20:28:20,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/57591d96c38f410f9cae101c9c724a3e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/57591d96c38f410f9cae101c9c724a3e 2024-12-04T20:28:20,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/57591d96c38f410f9cae101c9c724a3e, entries=150, sequenceid=303, filesize=12.0 K 2024-12-04T20:28:20,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1522ms, sequenceid=303, compaction requested=false 2024-12-04T20:28:20,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:20,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T20:28:20,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:20,191 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T20:28:20,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:20,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:20,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:20,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204abe1061effd9412eaea5996574cdc610_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344098674/Put/seqid=0 2024-12-04T20:28:20,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742029_1205 (size=12454) 2024-12-04T20:28:20,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204abe1061effd9412eaea5996574cdc610_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204abe1061effd9412eaea5996574cdc610_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/7a2fd06c414947d98270b93afa02e7bf, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/7a2fd06c414947d98270b93afa02e7bf is 175, key is test_row_0/A:col10/1733344098674/Put/seqid=0 2024-12-04T20:28:20,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742030_1206 (size=31255) 2024-12-04T20:28:20,285 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/7a2fd06c414947d98270b93afa02e7bf 2024-12-04T20:28:20,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/f206f355e75249138fe0f3b8a1f84d6e is 50, key is test_row_0/B:col10/1733344098674/Put/seqid=0 2024-12-04T20:28:20,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742031_1207 (size=12301) 2024-12-04T20:28:20,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/f206f355e75249138fe0f3b8a1f84d6e 2024-12-04T20:28:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/cf2d8f187be34f26945d8cead953c816 is 50, key is test_row_0/C:col10/1733344098674/Put/seqid=0 2024-12-04T20:28:20,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742032_1208 (size=12301) 2024-12-04T20:28:20,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/cf2d8f187be34f26945d8cead953c816 2024-12-04T20:28:20,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/7a2fd06c414947d98270b93afa02e7bf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/7a2fd06c414947d98270b93afa02e7bf 2024-12-04T20:28:20,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/7a2fd06c414947d98270b93afa02e7bf, entries=150, sequenceid=317, filesize=30.5 K 2024-12-04T20:28:20,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/f206f355e75249138fe0f3b8a1f84d6e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/f206f355e75249138fe0f3b8a1f84d6e 2024-12-04T20:28:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/f206f355e75249138fe0f3b8a1f84d6e, entries=150, sequenceid=317, filesize=12.0 K 2024-12-04T20:28:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/cf2d8f187be34f26945d8cead953c816 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf2d8f187be34f26945d8cead953c816 2024-12-04T20:28:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf2d8f187be34f26945d8cead953c816, entries=150, sequenceid=317, filesize=12.0 K 2024-12-04T20:28:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 608ms, sequenceid=317, compaction requested=true 2024-12-04T20:28:20,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:20,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-04T20:28:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-04T20:28:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-04T20:28:20,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8370 sec 2024-12-04T20:28:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8420 sec 2024-12-04T20:28:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:28:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:20,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120418dfa5a6a747437f83718e8479d28b05_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344100935/Put/seqid=0 2024-12-04T20:28:21,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742033_1209 (size=22618) 2024-12-04T20:28:21,036 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:21,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120418dfa5a6a747437f83718e8479d28b05_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418dfa5a6a747437f83718e8479d28b05_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:21,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/0dc7a0c6429a4525834a146ca27d270a, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:21,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/0dc7a0c6429a4525834a146ca27d270a is 175, key is test_row_0/A:col10/1733344100935/Put/seqid=0 2024-12-04T20:28:21,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742034_1210 (size=66019) 2024-12-04T20:28:21,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344101030, max timestamp=1733344101061Num uniques 859; 2024-12-04T20:28:21,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:28:21,070 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/0dc7a0c6429a4525834a146ca27d270a 2024-12-04T20:28:21,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/11c77b5e809149a783ff2840098263df is 50, key is test_row_0/B:col10/1733344100935/Put/seqid=0 2024-12-04T20:28:21,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344161094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344161097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:21,099 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:21,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742035_1211 (size=9857) 2024-12-04T20:28:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T20:28:21,132 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:21,132 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-04T20:28:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T20:28:21,136 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:21,136 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:21,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344161201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T20:28:21,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:21,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:21,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344161411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:21,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:21,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:21,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T20:28:21,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/11c77b5e809149a783ff2840098263df 2024-12-04T20:28:21,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/813e02bac3dd459eab6a4c2c5b982042 is 50, key is test_row_0/C:col10/1733344100935/Put/seqid=0 2024-12-04T20:28:21,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742036_1212 (size=9857) 2024-12-04T20:28:21,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/813e02bac3dd459eab6a4c2c5b982042 2024-12-04T20:28:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/0dc7a0c6429a4525834a146ca27d270a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/0dc7a0c6429a4525834a146ca27d270a 2024-12-04T20:28:21,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:21,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:21,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/0dc7a0c6429a4525834a146ca27d270a, entries=350, sequenceid=327, filesize=64.5 K 2024-12-04T20:28:21,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/11c77b5e809149a783ff2840098263df as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/11c77b5e809149a783ff2840098263df 2024-12-04T20:28:21,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/11c77b5e809149a783ff2840098263df, entries=100, sequenceid=327, filesize=9.6 K 2024-12-04T20:28:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/813e02bac3dd459eab6a4c2c5b982042 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/813e02bac3dd459eab6a4c2c5b982042 2024-12-04T20:28:21,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/813e02bac3dd459eab6a4c2c5b982042, entries=100, sequenceid=327, filesize=9.6 K 2024-12-04T20:28:21,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 663ms, sequenceid=327, compaction requested=true 2024-12-04T20:28:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:21,616 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:21,616 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:21,617 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160539 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:21,617 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:21,617 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,617 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:21,617 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:21,617 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/553798a837054c0dbab00c4e5d9457d9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/55861e0c123b4698959394ebdb0d9269, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/7a2fd06c414947d98270b93afa02e7bf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/0dc7a0c6429a4525834a146ca27d270a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=156.8 K 2024-12-04T20:28:21,617 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,617 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,617 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/553798a837054c0dbab00c4e5d9457d9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/55861e0c123b4698959394ebdb0d9269, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/7a2fd06c414947d98270b93afa02e7bf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/0dc7a0c6429a4525834a146ca27d270a] 2024-12-04T20:28:21,618 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/808706606ed8494cabdcf4841fc08785, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/166a5b755fab413bbfe17fd83068deee, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/f206f355e75249138fe0f3b8a1f84d6e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/11c77b5e809149a783ff2840098263df] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=46.3 K 2024-12-04T20:28:21,619 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553798a837054c0dbab00c4e5d9457d9, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733344096891 2024-12-04T20:28:21,619 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 808706606ed8494cabdcf4841fc08785, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733344096891 2024-12-04T20:28:21,619 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55861e0c123b4698959394ebdb0d9269, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733344098513 2024-12-04T20:28:21,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 166a5b755fab413bbfe17fd83068deee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733344098513 2024-12-04T20:28:21,620 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a2fd06c414947d98270b93afa02e7bf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733344098667 2024-12-04T20:28:21,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f206f355e75249138fe0f3b8a1f84d6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733344098667 2024-12-04T20:28:21,620 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc7a0c6429a4525834a146ca27d270a, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733344100898 2024-12-04T20:28:21,620 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c77b5e809149a783ff2840098263df, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733344100935 2024-12-04T20:28:21,636 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:21,636 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/75ba4f7cd56a447bb9bdd9463c848b07 is 50, key is test_row_0/B:col10/1733344100935/Put/seqid=0 2024-12-04T20:28:21,642 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:21,655 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204fa523adeb78f403d85b9604285e7edbc_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:21,657 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204fa523adeb78f403d85b9604285e7edbc_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:21,657 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fa523adeb78f403d85b9604285e7edbc_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:21,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742037_1213 (size=13085) 2024-12-04T20:28:21,670 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/75ba4f7cd56a447bb9bdd9463c848b07 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/75ba4f7cd56a447bb9bdd9463c848b07 2024-12-04T20:28:21,677 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into 75ba4f7cd56a447bb9bdd9463c848b07(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:21,677 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:21,678 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=12, startTime=1733344101616; duration=0sec 2024-12-04T20:28:21,678 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:21,678 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:21,678 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:21,680 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:21,681 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:21,681 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,681 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6bf3f086929c48e880dca2753c0062d2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/57591d96c38f410f9cae101c9c724a3e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf2d8f187be34f26945d8cead953c816, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/813e02bac3dd459eab6a4c2c5b982042] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=46.3 K 2024-12-04T20:28:21,681 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf3f086929c48e880dca2753c0062d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733344096891 2024-12-04T20:28:21,682 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 57591d96c38f410f9cae101c9c724a3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733344098513 2024-12-04T20:28:21,682 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2d8f187be34f26945d8cead953c816, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733344098667 2024-12-04T20:28:21,682 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 813e02bac3dd459eab6a4c2c5b982042, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733344100935 2024-12-04T20:28:21,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742038_1214 (size=4469) 2024-12-04T20:28:21,702 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#175 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:21,703 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/af21b1801b3d4f76b1b6fde1a5d037e9 is 175, key is test_row_0/A:col10/1733344100935/Put/seqid=0 2024-12-04T20:28:21,718 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:21,719 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/abb4a491ba854e31b9e8f4dbd61f4070 is 50, key is test_row_0/C:col10/1733344100935/Put/seqid=0 2024-12-04T20:28:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:21,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=138.65 KB heapSize=406.64 KB 2024-12-04T20:28:21,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:21,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:21,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742039_1215 (size=32146) 2024-12-04T20:28:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T20:28:21,764 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/af21b1801b3d4f76b1b6fde1a5d037e9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/af21b1801b3d4f76b1b6fde1a5d037e9 2024-12-04T20:28:21,772 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into af21b1801b3d4f76b1b6fde1a5d037e9(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:21,772 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:21,772 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=12, startTime=1733344101615; duration=0sec 2024-12-04T20:28:21,772 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:21,772 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:21,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049066a2f4d0704943a66461d2ac20e72a_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344101722/Put/seqid=0 2024-12-04T20:28:21,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742040_1216 (size=13085) 2024-12-04T20:28:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344161776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:21,790 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/abb4a491ba854e31b9e8f4dbd61f4070 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/abb4a491ba854e31b9e8f4dbd61f4070 2024-12-04T20:28:21,796 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into abb4a491ba854e31b9e8f4dbd61f4070(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:21,796 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:21,796 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=12, startTime=1733344101616; duration=0sec 2024-12-04T20:28:21,796 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:21,797 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:21,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742041_1217 (size=14994) 2024-12-04T20:28:21,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344161891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:21,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:21,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344162101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:22,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049066a2f4d0704943a66461d2ac20e72a_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049066a2f4d0704943a66461d2ac20e72a_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:22,224 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/bc319088216d4639bd0aa7808e167204, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:22,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/bc319088216d4639bd0aa7808e167204 is 175, key is test_row_0/A:col10/1733344101722/Put/seqid=0 2024-12-04T20:28:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742042_1218 (size=39949) 2024-12-04T20:28:22,251 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/bc319088216d4639bd0aa7808e167204 2024-12-04T20:28:22,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7be6296133de4fa48e9e9fc3b50dc4bb is 50, key is test_row_0/B:col10/1733344101722/Put/seqid=0 2024-12-04T20:28:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T20:28:22,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742043_1219 (size=12301) 2024-12-04T20:28:22,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344162412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:22,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:22,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7be6296133de4fa48e9e9fc3b50dc4bb 2024-12-04T20:28:22,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/2ab8b14539534733a1e1a383ba1820b6 is 50, key is test_row_0/C:col10/1733344101722/Put/seqid=0 2024-12-04T20:28:22,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742044_1220 (size=12301) 2024-12-04T20:28:22,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/2ab8b14539534733a1e1a383ba1820b6 2024-12-04T20:28:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/bc319088216d4639bd0aa7808e167204 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/bc319088216d4639bd0aa7808e167204 2024-12-04T20:28:22,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/bc319088216d4639bd0aa7808e167204, entries=200, sequenceid=356, filesize=39.0 K 2024-12-04T20:28:22,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/7be6296133de4fa48e9e9fc3b50dc4bb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7be6296133de4fa48e9e9fc3b50dc4bb 2024-12-04T20:28:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7be6296133de4fa48e9e9fc3b50dc4bb, entries=150, sequenceid=356, filesize=12.0 K 2024-12-04T20:28:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/2ab8b14539534733a1e1a383ba1820b6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2ab8b14539534733a1e1a383ba1820b6 2024-12-04T20:28:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2ab8b14539534733a1e1a383ba1820b6, entries=150, sequenceid=356, filesize=12.0 K 2024-12-04T20:28:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~406.59 KB/416352, currentSize=40.25 KB/41220 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1011ms, sequenceid=356, compaction requested=false 2024-12-04T20:28:22,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T20:28:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:22,829 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T20:28:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044a8b01273d8c418eaa1c033baeeac3ce_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344101748/Put/seqid=0 2024-12-04T20:28:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742045_1221 (size=9914) 2024-12-04T20:28:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,886 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044a8b01273d8c418eaa1c033baeeac3ce_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044a8b01273d8c418eaa1c033baeeac3ce_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/94a880c7446e435f829902dfd775d730, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/94a880c7446e435f829902dfd775d730 is 175, key is test_row_0/A:col10/1733344101748/Put/seqid=0 2024-12-04T20:28:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742046_1222 (size=22561) 2024-12-04T20:28:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. as already flushing 2024-12-04T20:28:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344103029, max timestamp=1733344103048Num uniques 859; 2024-12-04T20:28:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344103029, max timestamp=1733344103048Num uniques 859; 2024-12-04T20:28:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344103029, max timestamp=1733344103048Num uniques 859; 2024-12-04T20:28:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:23,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344103128, max timestamp=1733344103155Num uniques 1018; 2024-12-04T20:28:23,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T20:28:23,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344103128, max timestamp=1733344103155Num uniques 1018; 2024-12-04T20:28:23,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344103128, max timestamp=1733344103155Num uniques 1018; 2024-12-04T20:28:23,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T20:28:23,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T20:28:23,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344163226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T20:28:23,320 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/94a880c7446e435f829902dfd775d730 2024-12-04T20:28:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/41bebcedfd8b4047be0bd7187c88ce3c is 50, key is test_row_0/B:col10/1733344101748/Put/seqid=0 2024-12-04T20:28:23,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742047_1223 (size=9857) 2024-12-04T20:28:23,343 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/41bebcedfd8b4047be0bd7187c88ce3c 2024-12-04T20:28:23,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344163342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:23,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/cf17ae0e8f9946af9241b4497dc45eac is 50, key is test_row_0/C:col10/1733344101748/Put/seqid=0 2024-12-04T20:28:23,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742048_1224 (size=9857) 2024-12-04T20:28:23,372 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/cf17ae0e8f9946af9241b4497dc45eac 2024-12-04T20:28:23,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/94a880c7446e435f829902dfd775d730 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/94a880c7446e435f829902dfd775d730 2024-12-04T20:28:23,388 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/94a880c7446e435f829902dfd775d730, entries=100, sequenceid=367, filesize=22.0 K 2024-12-04T20:28:23,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/41bebcedfd8b4047be0bd7187c88ce3c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/41bebcedfd8b4047be0bd7187c88ce3c 2024-12-04T20:28:23,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/41bebcedfd8b4047be0bd7187c88ce3c, entries=100, sequenceid=367, filesize=9.6 K 2024-12-04T20:28:23,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/cf17ae0e8f9946af9241b4497dc45eac as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf17ae0e8f9946af9241b4497dc45eac 2024-12-04T20:28:23,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf17ae0e8f9946af9241b4497dc45eac, entries=100, sequenceid=367, filesize=9.6 K 2024-12-04T20:28:23,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 579ms, sequenceid=367, compaction requested=true 2024-12-04T20:28:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-04T20:28:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-04T20:28:23,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-04T20:28:23,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2720 sec 2024-12-04T20:28:23,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.2790 sec 2024-12-04T20:28:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:23,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-04T20:28:23,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:23,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:28:23,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:23,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:28:23,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:23,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:28:23,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c0ea247b4b4343279fc5f840d17e1e92_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344103224/Put/seqid=0 2024-12-04T20:28:23,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344163586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:23,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742049_1225 (size=14994) 2024-12-04T20:28:23,595 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:23,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c0ea247b4b4343279fc5f840d17e1e92_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c0ea247b4b4343279fc5f840d17e1e92_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:23,603 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/4eef59ba709149239a4eb6f442664b52, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:23,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/4eef59ba709149239a4eb6f442664b52 is 175, key is test_row_0/A:col10/1733344103224/Put/seqid=0 2024-12-04T20:28:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53756 deadline: 1733344163602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,606 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:23,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742050_1226 (size=39949) 2024-12-04T20:28:23,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53770 deadline: 1733344163622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,624 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:23,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53706 deadline: 1733344163641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,645 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18318 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344163701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:23,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344163912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:24,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/4eef59ba709149239a4eb6f442664b52 2024-12-04T20:28:24,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3f7d388e6e0a409da88ecde47cb8ec9e is 50, key is test_row_0/B:col10/1733344103224/Put/seqid=0 2024-12-04T20:28:24,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742051_1227 (size=12301) 2024-12-04T20:28:24,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3f7d388e6e0a409da88ecde47cb8ec9e 2024-12-04T20:28:24,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/e1e851f4c8fe4d258a781281ee229be7 is 50, key is test_row_0/C:col10/1733344103224/Put/seqid=0 2024-12-04T20:28:24,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742052_1228 (size=12301) 2024-12-04T20:28:24,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344164221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:24,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/e1e851f4c8fe4d258a781281ee229be7 2024-12-04T20:28:24,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/4eef59ba709149239a4eb6f442664b52 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/4eef59ba709149239a4eb6f442664b52 2024-12-04T20:28:24,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/4eef59ba709149239a4eb6f442664b52, entries=200, sequenceid=422, filesize=39.0 K 2024-12-04T20:28:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/3f7d388e6e0a409da88ecde47cb8ec9e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f7d388e6e0a409da88ecde47cb8ec9e 2024-12-04T20:28:24,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f7d388e6e0a409da88ecde47cb8ec9e, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T20:28:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/e1e851f4c8fe4d258a781281ee229be7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/e1e851f4c8fe4d258a781281ee229be7 2024-12-04T20:28:24,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/e1e851f4c8fe4d258a781281ee229be7, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T20:28:24,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1009ms, sequenceid=422, compaction requested=true 2024-12-04T20:28:24,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:24,564 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:24,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:24,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:24,565 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134605 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:24,566 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:24,566 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:24,566 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/af21b1801b3d4f76b1b6fde1a5d037e9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/bc319088216d4639bd0aa7808e167204, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/94a880c7446e435f829902dfd775d730, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/4eef59ba709149239a4eb6f442664b52] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=131.5 K 2024-12-04T20:28:24,566 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:24,566 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/af21b1801b3d4f76b1b6fde1a5d037e9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/bc319088216d4639bd0aa7808e167204, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/94a880c7446e435f829902dfd775d730, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/4eef59ba709149239a4eb6f442664b52] 2024-12-04T20:28:24,566 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:24,566 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting af21b1801b3d4f76b1b6fde1a5d037e9, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733344098667 2024-12-04T20:28:24,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:24,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:24,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:24,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:24,567 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc319088216d4639bd0aa7808e167204, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733344101083 2024-12-04T20:28:24,567 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a880c7446e435f829902dfd775d730, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733344101748 2024-12-04T20:28:24,567 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eef59ba709149239a4eb6f442664b52, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733344103205 2024-12-04T20:28:24,567 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:24,568 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:24,568 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:24,568 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/75ba4f7cd56a447bb9bdd9463c848b07, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7be6296133de4fa48e9e9fc3b50dc4bb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/41bebcedfd8b4047be0bd7187c88ce3c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f7d388e6e0a409da88ecde47cb8ec9e] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=46.4 K 2024-12-04T20:28:24,569 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ba4f7cd56a447bb9bdd9463c848b07, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733344098667 2024-12-04T20:28:24,569 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be6296133de4fa48e9e9fc3b50dc4bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733344101083 2024-12-04T20:28:24,570 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 41bebcedfd8b4047be0bd7187c88ce3c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733344101748 2024-12-04T20:28:24,571 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f7d388e6e0a409da88ecde47cb8ec9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733344103205 2024-12-04T20:28:24,582 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:24,587 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:24,588 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/5d07ad961bd04b53997aea6c5ab6612d is 50, key is test_row_0/B:col10/1733344103224/Put/seqid=0 2024-12-04T20:28:24,592 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412044e98385a293547eaafe7c93f8690e5e9_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:24,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412044e98385a293547eaafe7c93f8690e5e9_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:24,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044e98385a293547eaafe7c93f8690e5e9_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:24,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742054_1230 (size=4469) 2024-12-04T20:28:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742053_1229 (size=13221) 2024-12-04T20:28:24,670 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/5d07ad961bd04b53997aea6c5ab6612d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/5d07ad961bd04b53997aea6c5ab6612d 2024-12-04T20:28:24,677 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into 5d07ad961bd04b53997aea6c5ab6612d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:24,677 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:24,677 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=12, startTime=1733344104566; duration=0sec 2024-12-04T20:28:24,677 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:24,677 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:24,677 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:24,680 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:24,680 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:24,680 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:24,680 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/abb4a491ba854e31b9e8f4dbd61f4070, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2ab8b14539534733a1e1a383ba1820b6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf17ae0e8f9946af9241b4497dc45eac, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/e1e851f4c8fe4d258a781281ee229be7] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=46.4 K 2024-12-04T20:28:24,680 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting abb4a491ba854e31b9e8f4dbd61f4070, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733344098667 2024-12-04T20:28:24,681 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab8b14539534733a1e1a383ba1820b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733344101083 2024-12-04T20:28:24,681 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting cf17ae0e8f9946af9241b4497dc45eac, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733344101748 2024-12-04T20:28:24,682 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e851f4c8fe4d258a781281ee229be7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733344103205 2024-12-04T20:28:24,691 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:24,692 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/07d9ee50456640179702b84121bb9f9f is 50, key is test_row_0/C:col10/1733344103224/Put/seqid=0 2024-12-04T20:28:24,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742055_1231 (size=13221) 2024-12-04T20:28:24,709 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/07d9ee50456640179702b84121bb9f9f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/07d9ee50456640179702b84121bb9f9f 2024-12-04T20:28:24,716 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into 07d9ee50456640179702b84121bb9f9f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:24,716 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:24,716 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=12, startTime=1733344104567; duration=0sec 2024-12-04T20:28:24,716 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:24,716 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:24,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:28:24,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:24,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:24,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:24,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:24,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:24,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:24,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041683a317a686460a96856fbdffe46ae1_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344104735/Put/seqid=0 2024-12-04T20:28:24,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742056_1232 (size=12454) 2024-12-04T20:28:24,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344104790, max timestamp=1733344104832Num uniques 859; 2024-12-04T20:28:24,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:28:24,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344164858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344164962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:25,056 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#186 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:25,057 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/784faf6ccc7b492abdc2d8fc77e27701 is 175, key is test_row_0/A:col10/1733344103224/Put/seqid=0 2024-12-04T20:28:25,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742057_1233 (size=32175) 2024-12-04T20:28:25,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53744 deadline: 1733344165121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:25,124 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 8f15724cf3b6cc25ca4bfa788ed7a570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344165171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:25,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:25,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041683a317a686460a96856fbdffe46ae1_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041683a317a686460a96856fbdffe46ae1_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:25,188 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/49d1f12b9f1f4ebd81195708fb8ac44e, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:25,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/49d1f12b9f1f4ebd81195708fb8ac44e is 175, key is test_row_0/A:col10/1733344104735/Put/seqid=0 2024-12-04T20:28:25,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742058_1234 (size=31255) 2024-12-04T20:28:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T20:28:25,301 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-12-04T20:28:25,301 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:25,301 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,301 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:25,301 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:25,303 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-12-04T20:28:25,303 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:25,303 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,303 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:25,309 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-12-04T20:28:25,309 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:25,309 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,309 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:25,311 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-12-04T20:28:25,311 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:25,311 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,311 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:25,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:25,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344165482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:25,491 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/784faf6ccc7b492abdc2d8fc77e27701 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/784faf6ccc7b492abdc2d8fc77e27701 2024-12-04T20:28:25,494 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into 784faf6ccc7b492abdc2d8fc77e27701(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:25,494 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:25,494 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=12, startTime=1733344104564; duration=0sec 2024-12-04T20:28:25,495 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:25,495 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:25,603 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/49d1f12b9f1f4ebd81195708fb8ac44e 2024-12-04T20:28:25,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/eb62caaf4b9a4d8fa5e904802a697dbf is 50, key is test_row_0/B:col10/1733344104735/Put/seqid=0 2024-12-04T20:28:25,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742059_1235 (size=12301) 2024-12-04T20:28:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733344165992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:28:26,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/eb62caaf4b9a4d8fa5e904802a697dbf 2024-12-04T20:28:26,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/8cb1f3e178f44ee3be0e8645f2acc10f is 50, key is test_row_0/C:col10/1733344104735/Put/seqid=0 2024-12-04T20:28:26,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742060_1236 (size=12301) 2024-12-04T20:28:26,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/8cb1f3e178f44ee3be0e8645f2acc10f 2024-12-04T20:28:26,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/49d1f12b9f1f4ebd81195708fb8ac44e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/49d1f12b9f1f4ebd81195708fb8ac44e 2024-12-04T20:28:26,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/49d1f12b9f1f4ebd81195708fb8ac44e, entries=150, sequenceid=434, filesize=30.5 K 2024-12-04T20:28:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/eb62caaf4b9a4d8fa5e904802a697dbf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/eb62caaf4b9a4d8fa5e904802a697dbf 2024-12-04T20:28:26,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/eb62caaf4b9a4d8fa5e904802a697dbf, entries=150, sequenceid=434, filesize=12.0 K 2024-12-04T20:28:26,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/8cb1f3e178f44ee3be0e8645f2acc10f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8cb1f3e178f44ee3be0e8645f2acc10f 2024-12-04T20:28:26,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8cb1f3e178f44ee3be0e8645f2acc10f, entries=150, sequenceid=434, filesize=12.0 K 2024-12-04T20:28:26,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1704ms, sequenceid=434, compaction requested=false 2024-12-04T20:28:26,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:27,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=134.18 KB heapSize=391.80 KB 2024-12-04T20:28:27,003 INFO [Thread-618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-618. 2024-12-04T20:28:27,003 DEBUG [Thread-618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:27,003 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:27,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:27,003 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:27,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:27,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:27,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:27,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:27,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:27,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:27,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046e870138088941f5a2b44abefc5e2e0b_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344107002/Put/seqid=0 2024-12-04T20:28:27,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742061_1237 (size=12454) 2024-12-04T20:28:27,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:27,416 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046e870138088941f5a2b44abefc5e2e0b_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046e870138088941f5a2b44abefc5e2e0b_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:27,417 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d7296243f19d486abe6f8ea869862944, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:27,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d7296243f19d486abe6f8ea869862944 is 175, key is test_row_0/A:col10/1733344107002/Put/seqid=0 2024-12-04T20:28:27,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742062_1238 (size=31255) 2024-12-04T20:28:27,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=462, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d7296243f19d486abe6f8ea869862944 2024-12-04T20:28:27,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/b572d5b3431d4c398279492fe3ce73d9 is 50, key is test_row_0/B:col10/1733344107002/Put/seqid=0 2024-12-04T20:28:27,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742063_1239 (size=12301) 2024-12-04T20:28:27,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/b572d5b3431d4c398279492fe3ce73d9 2024-12-04T20:28:27,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/b55a1cf6047347e9b4ecc680a67df8d5 is 50, key is test_row_0/C:col10/1733344107002/Put/seqid=0 2024-12-04T20:28:27,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742064_1240 (size=12301) 2024-12-04T20:28:27,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/b55a1cf6047347e9b4ecc680a67df8d5 2024-12-04T20:28:27,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d7296243f19d486abe6f8ea869862944 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d7296243f19d486abe6f8ea869862944 2024-12-04T20:28:27,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d7296243f19d486abe6f8ea869862944, entries=150, sequenceid=462, filesize=30.5 K 2024-12-04T20:28:27,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/b572d5b3431d4c398279492fe3ce73d9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b572d5b3431d4c398279492fe3ce73d9 2024-12-04T20:28:27,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b572d5b3431d4c398279492fe3ce73d9, entries=150, sequenceid=462, filesize=12.0 K 2024-12-04T20:28:27,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/b55a1cf6047347e9b4ecc680a67df8d5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/b55a1cf6047347e9b4ecc680a67df8d5 2024-12-04T20:28:27,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/b55a1cf6047347e9b4ecc680a67df8d5, entries=150, sequenceid=462, filesize=12.0 K 2024-12-04T20:28:27,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~391.75 KB/401152, currentSize=0 B/0 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 453ms, sequenceid=462, compaction requested=true 2024-12-04T20:28:27,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:27,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:27,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:27,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:27,456 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f15724cf3b6cc25ca4bfa788ed7a570:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:27,457 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:27,457 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:27,457 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:27,457 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/A is initiating minor compaction (all files) 2024-12-04T20:28:27,457 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/B is initiating minor compaction (all files) 2024-12-04T20:28:27,457 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/A in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:27,458 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/B in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:27,458 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/5d07ad961bd04b53997aea6c5ab6612d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/eb62caaf4b9a4d8fa5e904802a697dbf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b572d5b3431d4c398279492fe3ce73d9] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=36.9 K 2024-12-04T20:28:27,458 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/784faf6ccc7b492abdc2d8fc77e27701, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/49d1f12b9f1f4ebd81195708fb8ac44e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d7296243f19d486abe6f8ea869862944] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=92.5 K 2024-12-04T20:28:27,458 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:27,458 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/784faf6ccc7b492abdc2d8fc77e27701, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/49d1f12b9f1f4ebd81195708fb8ac44e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d7296243f19d486abe6f8ea869862944] 2024-12-04T20:28:27,458 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d07ad961bd04b53997aea6c5ab6612d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733344103205 2024-12-04T20:28:27,458 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 784faf6ccc7b492abdc2d8fc77e27701, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733344103205 2024-12-04T20:28:27,458 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting eb62caaf4b9a4d8fa5e904802a697dbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733344103574 2024-12-04T20:28:27,458 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49d1f12b9f1f4ebd81195708fb8ac44e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733344103574 2024-12-04T20:28:27,458 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7296243f19d486abe6f8ea869862944, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733344104809 2024-12-04T20:28:27,458 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b572d5b3431d4c398279492fe3ce73d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733344104809 2024-12-04T20:28:27,465 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:27,466 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/8224e1e453c34419a8178e75cb32b547 is 50, key is test_row_0/B:col10/1733344107002/Put/seqid=0 2024-12-04T20:28:27,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742065_1241 (size=13323) 2024-12-04T20:28:27,469 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:27,475 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412044e4b65f5be2f48db8e057449c8150fb3_8f15724cf3b6cc25ca4bfa788ed7a570 store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:27,499 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412044e4b65f5be2f48db8e057449c8150fb3_8f15724cf3b6cc25ca4bfa788ed7a570, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:27,499 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044e4b65f5be2f48db8e057449c8150fb3_8f15724cf3b6cc25ca4bfa788ed7a570 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:27,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742066_1242 (size=4469) 2024-12-04T20:28:27,513 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#A#compaction#196 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:27,514 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/58695f363152496d9806014fe882cdf0 is 175, key is test_row_0/A:col10/1733344107002/Put/seqid=0 2024-12-04T20:28:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742067_1243 (size=32277) 2024-12-04T20:28:27,874 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/8224e1e453c34419a8178e75cb32b547 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/8224e1e453c34419a8178e75cb32b547 2024-12-04T20:28:27,879 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/B of 8f15724cf3b6cc25ca4bfa788ed7a570 into 8224e1e453c34419a8178e75cb32b547(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:27,879 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:27,879 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/B, priority=13, startTime=1733344107456; duration=0sec 2024-12-04T20:28:27,879 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:27,879 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:B 2024-12-04T20:28:27,879 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:27,880 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:27,880 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 8f15724cf3b6cc25ca4bfa788ed7a570/C is initiating minor compaction (all files) 2024-12-04T20:28:27,880 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f15724cf3b6cc25ca4bfa788ed7a570/C in TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:27,880 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/07d9ee50456640179702b84121bb9f9f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8cb1f3e178f44ee3be0e8645f2acc10f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/b55a1cf6047347e9b4ecc680a67df8d5] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp, totalSize=36.9 K 2024-12-04T20:28:27,881 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d9ee50456640179702b84121bb9f9f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733344103205 2024-12-04T20:28:27,881 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb1f3e178f44ee3be0e8645f2acc10f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733344103574 2024-12-04T20:28:27,881 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b55a1cf6047347e9b4ecc680a67df8d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733344104809 2024-12-04T20:28:27,888 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f15724cf3b6cc25ca4bfa788ed7a570#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:27,889 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/ef5d1f359de34e6bbe60bff1da342e2d is 50, key is test_row_0/C:col10/1733344107002/Put/seqid=0 2024-12-04T20:28:27,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742068_1244 (size=13323) 2024-12-04T20:28:27,899 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/ef5d1f359de34e6bbe60bff1da342e2d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/ef5d1f359de34e6bbe60bff1da342e2d 2024-12-04T20:28:27,903 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/C of 8f15724cf3b6cc25ca4bfa788ed7a570 into ef5d1f359de34e6bbe60bff1da342e2d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:27,903 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:27,903 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/C, priority=13, startTime=1733344107457; duration=0sec 2024-12-04T20:28:27,903 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:27,903 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:C 2024-12-04T20:28:27,928 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/58695f363152496d9806014fe882cdf0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/58695f363152496d9806014fe882cdf0 2024-12-04T20:28:27,932 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f15724cf3b6cc25ca4bfa788ed7a570/A of 8f15724cf3b6cc25ca4bfa788ed7a570 into 58695f363152496d9806014fe882cdf0(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:27,932 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f15724cf3b6cc25ca4bfa788ed7a570: 2024-12-04T20:28:27,932 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., storeName=8f15724cf3b6cc25ca4bfa788ed7a570/A, priority=13, startTime=1733344107456; duration=0sec 2024-12-04T20:28:27,932 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:27,932 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f15724cf3b6cc25ca4bfa788ed7a570:A 2024-12-04T20:28:32,786 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:28:33,633 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-04T20:28:33,633 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:33,633 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:33,633 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:33,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:33,713 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-04T20:28:33,713 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:33,713 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:33,713 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:33,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:33,725 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-12-04T20:28:33,725 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:33,725 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:33,725 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:33,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:35,192 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-04T20:28:35,192 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:35,192 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:35,193 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:35,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 2 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 258 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3124 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3080 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1358 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4074 rows 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1359 2024-12-04T20:28:35,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4077 rows 2024-12-04T20:28:35,193 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:28:35,193 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:28:35,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:35,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:35,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:35,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T20:28:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T20:28:35,197 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344115196"}]},"ts":"1733344115196"} 2024-12-04T20:28:35,198 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T20:28:35,198 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T20:28:35,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:28:35,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, UNASSIGN}] 2024-12-04T20:28:35,201 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, UNASSIGN 2024-12-04T20:28:35,201 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:35,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, UNASSIGN because future has completed 2024-12-04T20:28:35,202 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:28:35,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:28:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T20:28:35,355 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 8f15724cf3b6cc25ca4bfa788ed7a570, disabling compactions & flushes 2024-12-04T20:28:35,355 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. after waiting 0 ms 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:35,355 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 8f15724cf3b6cc25ca4bfa788ed7a570 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=A 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=B 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f15724cf3b6cc25ca4bfa788ed7a570, store=C 2024-12-04T20:28:35,355 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:35,360 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120443352d9802b44b379a73070f963a2d3c_8f15724cf3b6cc25ca4bfa788ed7a570 is 50, key is test_row_0/A:col10/1733344113722/Put/seqid=0 2024-12-04T20:28:35,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742069_1245 (size=12454) 2024-12-04T20:28:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T20:28:35,764 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:35,768 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120443352d9802b44b379a73070f963a2d3c_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120443352d9802b44b379a73070f963a2d3c_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:35,768 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d2fa4c73efcb4ee9a24f277191ab5c4d, store: [table=TestAcidGuarantees family=A region=8f15724cf3b6cc25ca4bfa788ed7a570] 2024-12-04T20:28:35,769 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d2fa4c73efcb4ee9a24f277191ab5c4d is 175, key is test_row_0/A:col10/1733344113722/Put/seqid=0 2024-12-04T20:28:35,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742070_1246 (size=31255) 2024-12-04T20:28:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T20:28:36,172 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d2fa4c73efcb4ee9a24f277191ab5c4d 2024-12-04T20:28:36,178 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/35917383af01493fb091e06316c802b3 is 50, key is test_row_0/B:col10/1733344113722/Put/seqid=0 2024-12-04T20:28:36,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742071_1247 (size=12301) 2024-12-04T20:28:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T20:28:36,582 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/35917383af01493fb091e06316c802b3 2024-12-04T20:28:36,588 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/2f94de392db44d3cba612351c4764465 is 50, key is test_row_0/C:col10/1733344113722/Put/seqid=0 2024-12-04T20:28:36,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742072_1248 (size=12301) 2024-12-04T20:28:36,992 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/2f94de392db44d3cba612351c4764465 2024-12-04T20:28:36,996 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/A/d2fa4c73efcb4ee9a24f277191ab5c4d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d2fa4c73efcb4ee9a24f277191ab5c4d 2024-12-04T20:28:36,999 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d2fa4c73efcb4ee9a24f277191ab5c4d, entries=150, sequenceid=472, filesize=30.5 K 2024-12-04T20:28:37,000 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/B/35917383af01493fb091e06316c802b3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/35917383af01493fb091e06316c802b3 2024-12-04T20:28:37,003 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/35917383af01493fb091e06316c802b3, entries=150, sequenceid=472, filesize=12.0 K 2024-12-04T20:28:37,004 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/.tmp/C/2f94de392db44d3cba612351c4764465 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2f94de392db44d3cba612351c4764465 2024-12-04T20:28:37,007 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2f94de392db44d3cba612351c4764465, entries=150, sequenceid=472, filesize=12.0 K 2024-12-04T20:28:37,008 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1653ms, sequenceid=472, compaction requested=false 2024-12-04T20:28:37,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c28f45bd5c7b406492e2da9c81b66b8c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/77b5c9a8e2724949aaf7af1c0473c02d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/8cdf6d19235e42a0b33a9c5ad03b1b74, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/1a5044f6941042f4a38950cbb0554730, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6beccb95ef6044c9957ddbae255a43aa, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/eac47076a9284ba7837a403cd7ccd0f4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/348d141c09bf4054b92efb73da51a8b2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/e1bc291aa35e46d4b31005f860efb18a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/374583fb511842688819039bc4029f2a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6658cf217321484ea0af8400f55237a9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/aaec981d9b32472dbde35d9b18c218c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c68e73266f6343f5b49f15c8bc6cbb28, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/a28edbbf2feb4d68880dc7af25e0003f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/71c713b6186c48f9a879fba2faec5f75, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/483ba75d41f046c097c666fe9335a1a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/524be53d7f2249af9302a2f4451979a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/f1a75af4a1eb4dc1bb790057648136fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/773f6e9e96ac4ceca3ddce83a1f1f5fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/553798a837054c0dbab00c4e5d9457d9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/673f49c1b40942b585cb7f369c591210, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/55861e0c123b4698959394ebdb0d9269, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/7a2fd06c414947d98270b93afa02e7bf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/0dc7a0c6429a4525834a146ca27d270a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/af21b1801b3d4f76b1b6fde1a5d037e9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/bc319088216d4639bd0aa7808e167204, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/94a880c7446e435f829902dfd775d730, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/4eef59ba709149239a4eb6f442664b52, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/784faf6ccc7b492abdc2d8fc77e27701, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/49d1f12b9f1f4ebd81195708fb8ac44e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d7296243f19d486abe6f8ea869862944] to archive 2024-12-04T20:28:37,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:28:37,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c28f45bd5c7b406492e2da9c81b66b8c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c28f45bd5c7b406492e2da9c81b66b8c 2024-12-04T20:28:37,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/77b5c9a8e2724949aaf7af1c0473c02d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/77b5c9a8e2724949aaf7af1c0473c02d 2024-12-04T20:28:37,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/8cdf6d19235e42a0b33a9c5ad03b1b74 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/8cdf6d19235e42a0b33a9c5ad03b1b74 2024-12-04T20:28:37,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/1a5044f6941042f4a38950cbb0554730 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/1a5044f6941042f4a38950cbb0554730 2024-12-04T20:28:37,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6beccb95ef6044c9957ddbae255a43aa to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6beccb95ef6044c9957ddbae255a43aa 2024-12-04T20:28:37,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/eac47076a9284ba7837a403cd7ccd0f4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/eac47076a9284ba7837a403cd7ccd0f4 2024-12-04T20:28:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/348d141c09bf4054b92efb73da51a8b2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/348d141c09bf4054b92efb73da51a8b2 2024-12-04T20:28:37,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/e1bc291aa35e46d4b31005f860efb18a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/e1bc291aa35e46d4b31005f860efb18a 2024-12-04T20:28:37,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/374583fb511842688819039bc4029f2a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/374583fb511842688819039bc4029f2a 2024-12-04T20:28:37,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6658cf217321484ea0af8400f55237a9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/6658cf217321484ea0af8400f55237a9 2024-12-04T20:28:37,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/aaec981d9b32472dbde35d9b18c218c2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/aaec981d9b32472dbde35d9b18c218c2 2024-12-04T20:28:37,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c68e73266f6343f5b49f15c8bc6cbb28 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/c68e73266f6343f5b49f15c8bc6cbb28 2024-12-04T20:28:37,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/a28edbbf2feb4d68880dc7af25e0003f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/a28edbbf2feb4d68880dc7af25e0003f 2024-12-04T20:28:37,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/71c713b6186c48f9a879fba2faec5f75 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/71c713b6186c48f9a879fba2faec5f75 2024-12-04T20:28:37,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/483ba75d41f046c097c666fe9335a1a4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/483ba75d41f046c097c666fe9335a1a4 2024-12-04T20:28:37,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/524be53d7f2249af9302a2f4451979a4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/524be53d7f2249af9302a2f4451979a4 2024-12-04T20:28:37,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/f1a75af4a1eb4dc1bb790057648136fd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/f1a75af4a1eb4dc1bb790057648136fd 2024-12-04T20:28:37,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/773f6e9e96ac4ceca3ddce83a1f1f5fd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/773f6e9e96ac4ceca3ddce83a1f1f5fd 2024-12-04T20:28:37,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/553798a837054c0dbab00c4e5d9457d9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/553798a837054c0dbab00c4e5d9457d9 2024-12-04T20:28:37,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/673f49c1b40942b585cb7f369c591210 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/673f49c1b40942b585cb7f369c591210 2024-12-04T20:28:37,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/55861e0c123b4698959394ebdb0d9269 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/55861e0c123b4698959394ebdb0d9269 2024-12-04T20:28:37,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/7a2fd06c414947d98270b93afa02e7bf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/7a2fd06c414947d98270b93afa02e7bf 2024-12-04T20:28:37,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/0dc7a0c6429a4525834a146ca27d270a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/0dc7a0c6429a4525834a146ca27d270a 2024-12-04T20:28:37,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/af21b1801b3d4f76b1b6fde1a5d037e9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/af21b1801b3d4f76b1b6fde1a5d037e9 2024-12-04T20:28:37,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/bc319088216d4639bd0aa7808e167204 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/bc319088216d4639bd0aa7808e167204 2024-12-04T20:28:37,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/94a880c7446e435f829902dfd775d730 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/94a880c7446e435f829902dfd775d730 2024-12-04T20:28:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/4eef59ba709149239a4eb6f442664b52 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/4eef59ba709149239a4eb6f442664b52 2024-12-04T20:28:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/784faf6ccc7b492abdc2d8fc77e27701 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/784faf6ccc7b492abdc2d8fc77e27701 2024-12-04T20:28:37,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/49d1f12b9f1f4ebd81195708fb8ac44e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/49d1f12b9f1f4ebd81195708fb8ac44e 2024-12-04T20:28:37,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d7296243f19d486abe6f8ea869862944 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d7296243f19d486abe6f8ea869862944 2024-12-04T20:28:37,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d2f17319f9c84e9d9be67e4f9f07a98f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7107d76993e1499a8e7575a0941154c7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/89717c6a23b24d59891960d6d326c6d5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3587dfe119574f2c937102eb55fb537e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7af2c4b1ae434aa186ac0ffef7b47ccb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d955e344fec14f21acba3210015f4592, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/91bbf633aa474e36ac19ec5bb5aa48ce, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/295c1d7dba814ee7b8cd33b66576824d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2aab068641b549d9b29349f2482ce06a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d6958e18a7aa4cf8b0979fc1b6871e57, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/154db25c19e84d30baf48e01f7a9a09b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f0ce740941c4f1792e05e0c6dbb2e7f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7849f3cf1b6549f994299e71507e6f6d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7772ed14aa474e6c83677d3985edb21c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2876738038684202b58317e42ba35bb1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b98cca193f754e88a1af931a3286f9fa, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/30c77e45e57146af85785f21982f26da, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/808706606ed8494cabdcf4841fc08785, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/de5e9b07fe244decaff0686242064085, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/166a5b755fab413bbfe17fd83068deee, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/f206f355e75249138fe0f3b8a1f84d6e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/75ba4f7cd56a447bb9bdd9463c848b07, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/11c77b5e809149a783ff2840098263df, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7be6296133de4fa48e9e9fc3b50dc4bb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/41bebcedfd8b4047be0bd7187c88ce3c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/5d07ad961bd04b53997aea6c5ab6612d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f7d388e6e0a409da88ecde47cb8ec9e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/eb62caaf4b9a4d8fa5e904802a697dbf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b572d5b3431d4c398279492fe3ce73d9] to archive 2024-12-04T20:28:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:28:37,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d2f17319f9c84e9d9be67e4f9f07a98f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d2f17319f9c84e9d9be67e4f9f07a98f 2024-12-04T20:28:37,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7107d76993e1499a8e7575a0941154c7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7107d76993e1499a8e7575a0941154c7 2024-12-04T20:28:37,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/89717c6a23b24d59891960d6d326c6d5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/89717c6a23b24d59891960d6d326c6d5 2024-12-04T20:28:37,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3587dfe119574f2c937102eb55fb537e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3587dfe119574f2c937102eb55fb537e 2024-12-04T20:28:37,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7af2c4b1ae434aa186ac0ffef7b47ccb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7af2c4b1ae434aa186ac0ffef7b47ccb 2024-12-04T20:28:37,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d955e344fec14f21acba3210015f4592 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d955e344fec14f21acba3210015f4592 2024-12-04T20:28:37,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/91bbf633aa474e36ac19ec5bb5aa48ce to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/91bbf633aa474e36ac19ec5bb5aa48ce 2024-12-04T20:28:37,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/295c1d7dba814ee7b8cd33b66576824d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/295c1d7dba814ee7b8cd33b66576824d 2024-12-04T20:28:37,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2aab068641b549d9b29349f2482ce06a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2aab068641b549d9b29349f2482ce06a 2024-12-04T20:28:37,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d6958e18a7aa4cf8b0979fc1b6871e57 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/d6958e18a7aa4cf8b0979fc1b6871e57 2024-12-04T20:28:37,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/154db25c19e84d30baf48e01f7a9a09b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/154db25c19e84d30baf48e01f7a9a09b 2024-12-04T20:28:37,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f0ce740941c4f1792e05e0c6dbb2e7f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f0ce740941c4f1792e05e0c6dbb2e7f 2024-12-04T20:28:37,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7849f3cf1b6549f994299e71507e6f6d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7849f3cf1b6549f994299e71507e6f6d 2024-12-04T20:28:37,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7772ed14aa474e6c83677d3985edb21c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7772ed14aa474e6c83677d3985edb21c 2024-12-04T20:28:37,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2876738038684202b58317e42ba35bb1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/2876738038684202b58317e42ba35bb1 2024-12-04T20:28:37,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b98cca193f754e88a1af931a3286f9fa to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b98cca193f754e88a1af931a3286f9fa 2024-12-04T20:28:37,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/30c77e45e57146af85785f21982f26da to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/30c77e45e57146af85785f21982f26da 2024-12-04T20:28:37,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/4cf6b5e16e3b49a89d37c0d37b2a1b3e 2024-12-04T20:28:37,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/808706606ed8494cabdcf4841fc08785 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/808706606ed8494cabdcf4841fc08785 2024-12-04T20:28:37,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/de5e9b07fe244decaff0686242064085 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/de5e9b07fe244decaff0686242064085 2024-12-04T20:28:37,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/166a5b755fab413bbfe17fd83068deee to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/166a5b755fab413bbfe17fd83068deee 2024-12-04T20:28:37,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/f206f355e75249138fe0f3b8a1f84d6e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/f206f355e75249138fe0f3b8a1f84d6e 2024-12-04T20:28:37,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/75ba4f7cd56a447bb9bdd9463c848b07 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/75ba4f7cd56a447bb9bdd9463c848b07 2024-12-04T20:28:37,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/11c77b5e809149a783ff2840098263df to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/11c77b5e809149a783ff2840098263df 2024-12-04T20:28:37,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7be6296133de4fa48e9e9fc3b50dc4bb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/7be6296133de4fa48e9e9fc3b50dc4bb 2024-12-04T20:28:37,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/41bebcedfd8b4047be0bd7187c88ce3c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/41bebcedfd8b4047be0bd7187c88ce3c 2024-12-04T20:28:37,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/5d07ad961bd04b53997aea6c5ab6612d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/5d07ad961bd04b53997aea6c5ab6612d 2024-12-04T20:28:37,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f7d388e6e0a409da88ecde47cb8ec9e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/3f7d388e6e0a409da88ecde47cb8ec9e 2024-12-04T20:28:37,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/eb62caaf4b9a4d8fa5e904802a697dbf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/eb62caaf4b9a4d8fa5e904802a697dbf 2024-12-04T20:28:37,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b572d5b3431d4c398279492fe3ce73d9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/b572d5b3431d4c398279492fe3ce73d9 2024-12-04T20:28:37,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/f69c0dbef35f44aeb552ae842282a868, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/48844e063e6c46fe89ca23888aa5aa64, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/eec16a23481d4bf88b9a5e79c010b85c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/1c796019afe04ae7ad611f7554fa65b2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/30d9dbb9896247208de8d7f10c1996d1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/872abad0c68c4a73bc45ce9cbcc7d8ff, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/82bc80acfb2c4fe3a74e76b6c181ceab, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/3bdba734bb1845a2856dfbae222bb5a4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/887b8472e69c42bf867e4d5063e58ee5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6be61be02c0140eab4d953dc30d5f7d6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/87aa56fbb9a64f43ae2a3e696caa31d7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8325384bc1604991908f101513c1873d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/363b9145e38a4486bcac73e72c4effa0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/80d2a421c22f41d0a3bf93e73b6dca81, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6ee536f193c941b58600ba8110cd554a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/42f971a0376544c282cb16628bafd8d1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/32b96d0350284aad9abcdbbe3bf585ec, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/69f077dcdda54092a13130448b150394, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6bf3f086929c48e880dca2753c0062d2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/73decd68acbe4acc9a8e04339e9f658d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/57591d96c38f410f9cae101c9c724a3e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf2d8f187be34f26945d8cead953c816, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/abb4a491ba854e31b9e8f4dbd61f4070, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/813e02bac3dd459eab6a4c2c5b982042, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2ab8b14539534733a1e1a383ba1820b6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf17ae0e8f9946af9241b4497dc45eac, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/07d9ee50456640179702b84121bb9f9f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/e1e851f4c8fe4d258a781281ee229be7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8cb1f3e178f44ee3be0e8645f2acc10f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/b55a1cf6047347e9b4ecc680a67df8d5] to archive 2024-12-04T20:28:37,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:28:37,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/f69c0dbef35f44aeb552ae842282a868 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/f69c0dbef35f44aeb552ae842282a868 2024-12-04T20:28:37,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/48844e063e6c46fe89ca23888aa5aa64 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/48844e063e6c46fe89ca23888aa5aa64 2024-12-04T20:28:37,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/eec16a23481d4bf88b9a5e79c010b85c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/eec16a23481d4bf88b9a5e79c010b85c 2024-12-04T20:28:37,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/1c796019afe04ae7ad611f7554fa65b2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/1c796019afe04ae7ad611f7554fa65b2 2024-12-04T20:28:37,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/30d9dbb9896247208de8d7f10c1996d1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/30d9dbb9896247208de8d7f10c1996d1 2024-12-04T20:28:37,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/872abad0c68c4a73bc45ce9cbcc7d8ff to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/872abad0c68c4a73bc45ce9cbcc7d8ff 2024-12-04T20:28:37,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/82bc80acfb2c4fe3a74e76b6c181ceab to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/82bc80acfb2c4fe3a74e76b6c181ceab 2024-12-04T20:28:37,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/3bdba734bb1845a2856dfbae222bb5a4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/3bdba734bb1845a2856dfbae222bb5a4 2024-12-04T20:28:37,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/887b8472e69c42bf867e4d5063e58ee5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/887b8472e69c42bf867e4d5063e58ee5 2024-12-04T20:28:37,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6be61be02c0140eab4d953dc30d5f7d6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6be61be02c0140eab4d953dc30d5f7d6 2024-12-04T20:28:37,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/87aa56fbb9a64f43ae2a3e696caa31d7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/87aa56fbb9a64f43ae2a3e696caa31d7 2024-12-04T20:28:37,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8325384bc1604991908f101513c1873d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8325384bc1604991908f101513c1873d 2024-12-04T20:28:37,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/363b9145e38a4486bcac73e72c4effa0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/363b9145e38a4486bcac73e72c4effa0 2024-12-04T20:28:37,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/80d2a421c22f41d0a3bf93e73b6dca81 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/80d2a421c22f41d0a3bf93e73b6dca81 2024-12-04T20:28:37,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6ee536f193c941b58600ba8110cd554a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6ee536f193c941b58600ba8110cd554a 2024-12-04T20:28:37,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/42f971a0376544c282cb16628bafd8d1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/42f971a0376544c282cb16628bafd8d1 2024-12-04T20:28:37,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/32b96d0350284aad9abcdbbe3bf585ec to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/32b96d0350284aad9abcdbbe3bf585ec 2024-12-04T20:28:37,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/69f077dcdda54092a13130448b150394 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/69f077dcdda54092a13130448b150394 2024-12-04T20:28:37,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6bf3f086929c48e880dca2753c0062d2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/6bf3f086929c48e880dca2753c0062d2 2024-12-04T20:28:37,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/73decd68acbe4acc9a8e04339e9f658d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/73decd68acbe4acc9a8e04339e9f658d 2024-12-04T20:28:37,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/57591d96c38f410f9cae101c9c724a3e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/57591d96c38f410f9cae101c9c724a3e 2024-12-04T20:28:37,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf2d8f187be34f26945d8cead953c816 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf2d8f187be34f26945d8cead953c816 2024-12-04T20:28:37,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/abb4a491ba854e31b9e8f4dbd61f4070 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/abb4a491ba854e31b9e8f4dbd61f4070 2024-12-04T20:28:37,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/813e02bac3dd459eab6a4c2c5b982042 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/813e02bac3dd459eab6a4c2c5b982042 2024-12-04T20:28:37,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2ab8b14539534733a1e1a383ba1820b6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2ab8b14539534733a1e1a383ba1820b6 2024-12-04T20:28:37,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf17ae0e8f9946af9241b4497dc45eac to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/cf17ae0e8f9946af9241b4497dc45eac 2024-12-04T20:28:37,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/07d9ee50456640179702b84121bb9f9f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/07d9ee50456640179702b84121bb9f9f 2024-12-04T20:28:37,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/e1e851f4c8fe4d258a781281ee229be7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/e1e851f4c8fe4d258a781281ee229be7 2024-12-04T20:28:37,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8cb1f3e178f44ee3be0e8645f2acc10f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/8cb1f3e178f44ee3be0e8645f2acc10f 2024-12-04T20:28:37,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/b55a1cf6047347e9b4ecc680a67df8d5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/b55a1cf6047347e9b4ecc680a67df8d5 2024-12-04T20:28:37,106 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/recovered.edits/475.seqid, newMaxSeqId=475, maxSeqId=4 2024-12-04T20:28:37,107 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. 2024-12-04T20:28:37,107 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 8f15724cf3b6cc25ca4bfa788ed7a570: Waiting for close lock at 1733344115355Running coprocessor pre-close hooks at 1733344115355Disabling compacts and flushes for region at 1733344115355Disabling writes for close at 1733344115355Obtaining lock to block concurrent updates at 1733344115355Preparing flush snapshotting stores in 8f15724cf3b6cc25ca4bfa788ed7a570 at 1733344115355Finished memstore snapshotting TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733344115355Flushing stores of TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570. at 1733344115356 (+1 ms)Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/A: creating writer at 1733344115356Flushing mob file 8f15724cf3b6cc25ca4bfa788ed7a570/A: appending metadata at 1733344115360 (+4 ms)Flushing mob file 8f15724cf3b6cc25ca4bfa788ed7a570/A: closing flushed file at 1733344115360Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/A: appending metadata at 1733344115768 (+408 ms)Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/A: closing flushed file at 1733344115769 (+1 ms)Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/B: creating writer at 1733344116176 (+407 ms)Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/B: appending metadata at 1733344116178 (+2 ms)Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/B: closing flushed file at 1733344116178Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/C: creating writer at 1733344116585 (+407 ms)Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/C: appending metadata at 1733344116588 (+3 ms)Flushing 8f15724cf3b6cc25ca4bfa788ed7a570/C: closing flushed file at 1733344116588Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7862096f: reopening flushed file at 1733344116996 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ab30f4: reopening flushed file at 1733344116999 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a3aec58: reopening flushed file at 1733344117003 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8f15724cf3b6cc25ca4bfa788ed7a570 in 1653ms, sequenceid=472, compaction requested=false at 1733344117008 (+5 ms)Writing region close event to WAL at 1733344117103 (+95 ms)Running coprocessor post-close hooks at 1733344117107 (+4 ms)Closed at 1733344117107 2024-12-04T20:28:37,109 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,109 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=8f15724cf3b6cc25ca4bfa788ed7a570, regionState=CLOSED 2024-12-04T20:28:37,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:28:37,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-04T20:28:37,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f15724cf3b6cc25ca4bfa788ed7a570, server=c2ef38372881,45443,1733344054291 in 1.9080 sec 2024-12-04T20:28:37,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-04T20:28:37,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f15724cf3b6cc25ca4bfa788ed7a570, UNASSIGN in 1.9130 sec 2024-12-04T20:28:37,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-04T20:28:37,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9160 sec 2024-12-04T20:28:37,118 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344117118"}]},"ts":"1733344117118"} 2024-12-04T20:28:37,120 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T20:28:37,120 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T20:28:37,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9260 sec 2024-12-04T20:28:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T20:28:37,342 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:37,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T20:28:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:37,344 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T20:28:37,344 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:37,347 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,349 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/recovered.edits] 2024-12-04T20:28:37,351 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/58695f363152496d9806014fe882cdf0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/58695f363152496d9806014fe882cdf0 2024-12-04T20:28:37,352 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d2fa4c73efcb4ee9a24f277191ab5c4d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/A/d2fa4c73efcb4ee9a24f277191ab5c4d 2024-12-04T20:28:37,354 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/35917383af01493fb091e06316c802b3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/35917383af01493fb091e06316c802b3 2024-12-04T20:28:37,355 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/8224e1e453c34419a8178e75cb32b547 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/B/8224e1e453c34419a8178e75cb32b547 2024-12-04T20:28:37,357 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2f94de392db44d3cba612351c4764465 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/2f94de392db44d3cba612351c4764465 2024-12-04T20:28:37,358 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/ef5d1f359de34e6bbe60bff1da342e2d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/C/ef5d1f359de34e6bbe60bff1da342e2d 2024-12-04T20:28:37,360 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/recovered.edits/475.seqid to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570/recovered.edits/475.seqid 2024-12-04T20:28:37,360 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T20:28:37,360 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T20:28:37,361 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T20:28:37,364 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120406b4efe9311e4695b68468cef40263b5_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120406b4efe9311e4695b68468cef40263b5_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,365 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040fb9990bffd745db8a061437b2b3407b_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040fb9990bffd745db8a061437b2b3407b_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,366 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041683a317a686460a96856fbdffe46ae1_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041683a317a686460a96856fbdffe46ae1_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,367 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418dfa5a6a747437f83718e8479d28b05_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418dfa5a6a747437f83718e8479d28b05_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,367 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041a290c24380840d1b6bd9e5f83caba8a_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041a290c24380840d1b6bd9e5f83caba8a_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,368 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204261f01f54bb44acdb9c9d9b1310f8a12_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204261f01f54bb44acdb9c9d9b1310f8a12_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,369 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042edf35d14c944fef8346bb54244bad95_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042edf35d14c944fef8346bb54244bad95_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,370 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204427b819cafa3449e8817cb71577c009e_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204427b819cafa3449e8817cb71577c009e_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,371 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120443352d9802b44b379a73070f963a2d3c_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120443352d9802b44b379a73070f963a2d3c_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,372 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044a8b01273d8c418eaa1c033baeeac3ce_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044a8b01273d8c418eaa1c033baeeac3ce_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,373 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045f3d47f0020748a7856035b0097942f0_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045f3d47f0020748a7856035b0097942f0_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,374 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046e870138088941f5a2b44abefc5e2e0b_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046e870138088941f5a2b44abefc5e2e0b_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,375 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204757c89101a8848c8a4fa243f5b2e3d08_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204757c89101a8848c8a4fa243f5b2e3d08_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,376 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047b903faf40454eb8b2870ebafc91f5e1_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047b903faf40454eb8b2870ebafc91f5e1_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,376 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047be1cdb0ae54498e8a89537ddfdbd403_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047be1cdb0ae54498e8a89537ddfdbd403_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,377 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120480b224e9642f4940b66be7914bae3158_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120480b224e9642f4940b66be7914bae3158_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,378 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204880d752703af43e5aceab0ce37c66c2e_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204880d752703af43e5aceab0ce37c66c2e_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,379 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049066a2f4d0704943a66461d2ac20e72a_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049066a2f4d0704943a66461d2ac20e72a_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,380 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a523ba1638e24d198ef65182cd015c4e_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a523ba1638e24d198ef65182cd015c4e_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,381 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204abe1061effd9412eaea5996574cdc610_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204abe1061effd9412eaea5996574cdc610_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,382 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bcd7c617f42f485cba239465963b2186_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bcd7c617f42f485cba239465963b2186_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,383 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c0ea247b4b4343279fc5f840d17e1e92_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c0ea247b4b4343279fc5f840d17e1e92_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,384 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ef4a3c5703d74ad38b8ae8be58d7bfaf_8f15724cf3b6cc25ca4bfa788ed7a570 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ef4a3c5703d74ad38b8ae8be58d7bfaf_8f15724cf3b6cc25ca4bfa788ed7a570 2024-12-04T20:28:37,384 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T20:28:37,386 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:37,389 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T20:28:37,391 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T20:28:37,392 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:37,392 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T20:28:37,393 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733344117393"}]},"ts":"9223372036854775807"} 2024-12-04T20:28:37,395 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T20:28:37,395 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8f15724cf3b6cc25ca4bfa788ed7a570, NAME => 'TestAcidGuarantees,,1733344082083.8f15724cf3b6cc25ca4bfa788ed7a570.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T20:28:37,395 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T20:28:37,395 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733344117395"}]},"ts":"9223372036854775807"} 2024-12-04T20:28:37,396 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T20:28:37,397 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:37,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-12-04T20:28:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T20:28:37,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T20:28:37,452 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:37,471 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 250), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=548 (was 458) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3719 (was 4055) 2024-12-04T20:28:37,479 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=548, ProcessCount=11, AvailableMemoryMB=3719 2024-12-04T20:28:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:28:37,481 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:28:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T20:28:37,483 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:28:37,483 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:37,483 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-04T20:28:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T20:28:37,484 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:28:37,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742073_1249 (size=1061) 2024-12-04T20:28:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T20:28:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T20:28:37,898 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:28:37,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742074_1250 (size=53) 2024-12-04T20:28:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T20:28:38,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:28:38,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2687aa8f5064ce602682f5c7138994be, disabling compactions & flushes 2024-12-04T20:28:38,304 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. after waiting 0 ms 2024-12-04T20:28:38,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,304 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2687aa8f5064ce602682f5c7138994be: Waiting for close lock at 1733344118304Disabling compacts and flushes for region at 1733344118304Disabling writes for close at 1733344118304Writing region close event to WAL at 1733344118304Closed at 1733344118304 2024-12-04T20:28:38,305 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:28:38,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733344118305"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733344118305"}]},"ts":"1733344118305"} 2024-12-04T20:28:38,307 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:28:38,308 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:28:38,308 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344118308"}]},"ts":"1733344118308"} 2024-12-04T20:28:38,309 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T20:28:38,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, ASSIGN}] 2024-12-04T20:28:38,310 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, ASSIGN 2024-12-04T20:28:38,311 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, ASSIGN; state=OFFLINE, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=false 2024-12-04T20:28:38,461 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=2687aa8f5064ce602682f5c7138994be, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:38,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, ASSIGN because future has completed 2024-12-04T20:28:38,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:28:38,618 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,618 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:28:38,618 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,618 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:28:38,618 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,618 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,619 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,620 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:38,621 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2687aa8f5064ce602682f5c7138994be columnFamilyName A 2024-12-04T20:28:38,621 DEBUG [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:38,621 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.HStore(327): Store=2687aa8f5064ce602682f5c7138994be/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:38,621 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T20:28:38,622 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:38,622 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2687aa8f5064ce602682f5c7138994be columnFamilyName B 2024-12-04T20:28:38,622 DEBUG [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:38,623 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.HStore(327): Store=2687aa8f5064ce602682f5c7138994be/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:38,623 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,624 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:28:38,624 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2687aa8f5064ce602682f5c7138994be columnFamilyName C 2024-12-04T20:28:38,624 DEBUG [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:28:38,624 INFO [StoreOpener-2687aa8f5064ce602682f5c7138994be-1 {}] regionserver.HStore(327): Store=2687aa8f5064ce602682f5c7138994be/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:28:38,625 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,625 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,625 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,625 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,626 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,626 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,627 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:28:38,628 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,629 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:28:38,629 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 2687aa8f5064ce602682f5c7138994be; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61112116, jitterRate=-0.08935850858688354}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:28:38,629 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:38,630 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 2687aa8f5064ce602682f5c7138994be: Running coprocessor pre-open hook at 1733344118618Writing region info on filesystem at 1733344118618Initializing all the Stores at 1733344118619 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344118619Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344118619Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344118619Cleaning up temporary data from old regions at 1733344118626 (+7 ms)Running coprocessor post-open hooks at 1733344118629 (+3 ms)Region opened successfully at 1733344118630 (+1 ms) 2024-12-04T20:28:38,631 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., pid=61, masterSystemTime=1733344118615 2024-12-04T20:28:38,633 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,633 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:38,634 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=2687aa8f5064ce602682f5c7138994be, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:28:38,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:28:38,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-04T20:28:38,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 in 172 msec 2024-12-04T20:28:38,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-04T20:28:38,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, ASSIGN in 328 msec 2024-12-04T20:28:38,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:28:38,639 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344118639"}]},"ts":"1733344118639"} 2024-12-04T20:28:38,640 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T20:28:38,641 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:28:38,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-12-04T20:28:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T20:28:39,642 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:39,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10656c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,646 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@312faf78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,649 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:28:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,43507,1733344053664 2024-12-04T20:28:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1debf58d 2024-12-04T20:28:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:28:39,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:28:39,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,652 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625c8d57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd6fbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,655 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e386816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,656 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c85a7d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c512a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,658 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4815c616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6770b4c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,661 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d92fa45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1216fc90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,663 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@974120b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14453eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ec798a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d68a83b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a635bfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d509c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656975f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,672 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17edb5a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,674 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344d77a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:28:39,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:28:39,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:28:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:28:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:28:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e00fad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:28:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:28:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:39,676 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:28:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac8b167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:28:39,677 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,677 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,677 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,677 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-04T20:28:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T20:28:39,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:39,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:39,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,681 DEBUG [Thread-1176 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,681 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,682 DEBUG [Thread-1178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,682 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,686 DEBUG [Thread-1159 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,686 DEBUG [Thread-1159 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,686 DEBUG [Thread-1167 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,690 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:39,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:28:39,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:39,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:39,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:39,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:39,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:39,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:39,713 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:28:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:28:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:28:39,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:28:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:28:39,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488 is 50, key is test_row_0/A:col10/1733344119696/Put/seqid=0 2024-12-04T20:28:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344179733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344179734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344119705, max timestamp=1733344119739Num uniques 859; 2024-12-04T20:28:39,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344119705, max timestamp=1733344119739Num uniques 859; 2024-12-04T20:28:39,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344119705, max timestamp=1733344119739Num uniques 859; 2024-12-04T20:28:39,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:28:39,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:28:39,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:28:39,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344179737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742075_1251 (size=12001) 2024-12-04T20:28:39,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488 2024-12-04T20:28:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T20:28:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344179785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344179788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/4874b39f1fe24013b793050e1f6c7924 is 50, key is test_row_0/B:col10/1733344119696/Put/seqid=0 2024-12-04T20:28:39,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742076_1252 (size=12001) 2024-12-04T20:28:39,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/4874b39f1fe24013b793050e1f6c7924 2024-12-04T20:28:39,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T20:28:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:39,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:39,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344179841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/21c72121fb894daea4b3ab69e42e17cd is 50, key is test_row_0/C:col10/1733344119696/Put/seqid=0 2024-12-04T20:28:39,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344179852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344179852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742077_1253 (size=12001) 2024-12-04T20:28:39,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/21c72121fb894daea4b3ab69e42e17cd 2024-12-04T20:28:39,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488 2024-12-04T20:28:39,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T20:28:39,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344179891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/4874b39f1fe24013b793050e1f6c7924 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/4874b39f1fe24013b793050e1f6c7924 2024-12-04T20:28:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344179902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:39,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/4874b39f1fe24013b793050e1f6c7924, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T20:28:39,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/21c72121fb894daea4b3ab69e42e17cd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/21c72121fb894daea4b3ab69e42e17cd 2024-12-04T20:28:39,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/21c72121fb894daea4b3ab69e42e17cd, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T20:28:39,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 2687aa8f5064ce602682f5c7138994be in 217ms, sequenceid=13, compaction requested=false 2024-12-04T20:28:39,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:39,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T20:28:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:39,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-04T20:28:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T20:28:40,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/e257a7b29f004c71ab4a720095870f5e is 50, key is test_row_0/A:col10/1733344119777/Put/seqid=0 2024-12-04T20:28:40,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:40,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742078_1254 (size=12001) 2024-12-04T20:28:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344180079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344180081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344180081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344180102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344180112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344180191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344180191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344180202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T20:28:40,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344180402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344180402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344180411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344180412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344180422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,457 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/e257a7b29f004c71ab4a720095870f5e 2024-12-04T20:28:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/c4c9867f607c4a32a479682437026279 is 50, key is test_row_0/B:col10/1733344119777/Put/seqid=0 2024-12-04T20:28:40,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742079_1255 (size=12001) 2024-12-04T20:28:40,487 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/c4c9867f607c4a32a479682437026279 2024-12-04T20:28:40,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/da0a6577193845e2b3764d000c51173e is 50, key is test_row_0/C:col10/1733344119777/Put/seqid=0 2024-12-04T20:28:40,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742080_1256 (size=12001) 2024-12-04T20:28:40,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/da0a6577193845e2b3764d000c51173e 2024-12-04T20:28:40,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/e257a7b29f004c71ab4a720095870f5e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/e257a7b29f004c71ab4a720095870f5e 2024-12-04T20:28:40,524 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/e257a7b29f004c71ab4a720095870f5e, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T20:28:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/c4c9867f607c4a32a479682437026279 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/c4c9867f607c4a32a479682437026279 2024-12-04T20:28:40,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/c4c9867f607c4a32a479682437026279, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T20:28:40,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/da0a6577193845e2b3764d000c51173e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/da0a6577193845e2b3764d000c51173e 2024-12-04T20:28:40,544 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/da0a6577193845e2b3764d000c51173e, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T20:28:40,548 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 2687aa8f5064ce602682f5c7138994be in 558ms, sequenceid=47, compaction requested=false 2024-12-04T20:28:40,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:40,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:40,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-04T20:28:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-04T20:28:40,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-04T20:28:40,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 870 msec 2024-12-04T20:28:40,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 877 msec 2024-12-04T20:28:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:40,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:28:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:40,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/60d66a22555045e3a03ca85d79788700 is 50, key is test_row_0/A:col10/1733344120713/Put/seqid=0 2024-12-04T20:28:40,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344180736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344180737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344180739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742081_1257 (size=14341) 2024-12-04T20:28:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T20:28:40,812 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:40,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-04T20:28:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T20:28:40,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:40,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:40,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:40,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344180852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344180852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344180852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T20:28:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344180922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344180931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:40,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:40,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:40,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:40,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:40,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:40,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344181062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344181062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344181062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:41,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:41,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T20:28:41,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/60d66a22555045e3a03ca85d79788700 2024-12-04T20:28:41,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/40a8391af55244f6862501cfcf94bc7c is 50, key is test_row_0/B:col10/1733344120713/Put/seqid=0 2024-12-04T20:28:41,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742082_1258 (size=12001) 2024-12-04T20:28:41,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/40a8391af55244f6862501cfcf94bc7c 2024-12-04T20:28:41,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/e52ae9fbba564f50b27ea0235db98a66 is 50, key is test_row_0/C:col10/1733344120713/Put/seqid=0 2024-12-04T20:28:41,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742083_1259 (size=12001) 2024-12-04T20:28:41,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/e52ae9fbba564f50b27ea0235db98a66 2024-12-04T20:28:41,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/60d66a22555045e3a03ca85d79788700 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/60d66a22555045e3a03ca85d79788700 2024-12-04T20:28:41,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:41,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/60d66a22555045e3a03ca85d79788700, entries=200, sequenceid=61, filesize=14.0 K 2024-12-04T20:28:41,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/40a8391af55244f6862501cfcf94bc7c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/40a8391af55244f6862501cfcf94bc7c 2024-12-04T20:28:41,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/40a8391af55244f6862501cfcf94bc7c, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T20:28:41,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/e52ae9fbba564f50b27ea0235db98a66 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e52ae9fbba564f50b27ea0235db98a66 2024-12-04T20:28:41,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e52ae9fbba564f50b27ea0235db98a66, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T20:28:41,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2687aa8f5064ce602682f5c7138994be in 599ms, sequenceid=61, compaction requested=true 2024-12-04T20:28:41,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:41,312 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:41,313 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:41,314 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:41,314 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:41,314 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,314 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/e257a7b29f004c71ab4a720095870f5e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/60d66a22555045e3a03ca85d79788700] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=37.4 K 2024-12-04T20:28:41,314 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:41,314 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:41,314 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,314 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f69d7e8d9c49e48a6f3fd5bf7f2488, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733344119692 2024-12-04T20:28:41,314 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/4874b39f1fe24013b793050e1f6c7924, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/c4c9867f607c4a32a479682437026279, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/40a8391af55244f6862501cfcf94bc7c] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=35.2 K 2024-12-04T20:28:41,315 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 4874b39f1fe24013b793050e1f6c7924, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733344119692 2024-12-04T20:28:41,315 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e257a7b29f004c71ab4a720095870f5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733344119777 2024-12-04T20:28:41,315 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60d66a22555045e3a03ca85d79788700, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344120069 2024-12-04T20:28:41,315 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c9867f607c4a32a479682437026279, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733344119777 2024-12-04T20:28:41,316 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a8391af55244f6862501cfcf94bc7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344120079 2024-12-04T20:28:41,337 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:41,337 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/21adbc200940479f90fbb20dfcb4e7bd is 50, key is test_row_0/A:col10/1733344120713/Put/seqid=0 2024-12-04T20:28:41,347 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:41,347 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/6d96d2fbfacb4788af30cf9c91d8a1a0 is 50, key is test_row_0/B:col10/1733344120713/Put/seqid=0 2024-12-04T20:28:41,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742085_1261 (size=12104) 2024-12-04T20:28:41,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742084_1260 (size=12104) 2024-12-04T20:28:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:41,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T20:28:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:41,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d655e27423904899992402a7bb2b35c2 is 50, key is test_row_0/A:col10/1733344121372/Put/seqid=0 2024-12-04T20:28:41,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742086_1262 (size=12001) 2024-12-04T20:28:41,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344181391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344181391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344181392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:41,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T20:28:41,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344181502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344181502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344181502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:41,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:41,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344181712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344181712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344181712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:41,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:41,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,759 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/6d96d2fbfacb4788af30cf9c91d8a1a0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/6d96d2fbfacb4788af30cf9c91d8a1a0 2024-12-04T20:28:41,764 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into 6d96d2fbfacb4788af30cf9c91d8a1a0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:41,764 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:41,764 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344121313; duration=0sec 2024-12-04T20:28:41,764 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:41,764 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:41,764 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:41,766 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:41,766 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:41,766 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,766 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/21c72121fb894daea4b3ab69e42e17cd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/da0a6577193845e2b3764d000c51173e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e52ae9fbba564f50b27ea0235db98a66] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=35.2 K 2024-12-04T20:28:41,767 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c72121fb894daea4b3ab69e42e17cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733344119692 2024-12-04T20:28:41,768 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting da0a6577193845e2b3764d000c51173e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733344119777 2024-12-04T20:28:41,768 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/21adbc200940479f90fbb20dfcb4e7bd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21adbc200940479f90fbb20dfcb4e7bd 2024-12-04T20:28:41,768 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e52ae9fbba564f50b27ea0235db98a66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344120079 2024-12-04T20:28:41,772 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into 21adbc200940479f90fbb20dfcb4e7bd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:41,772 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:41,772 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344121312; duration=0sec 2024-12-04T20:28:41,772 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:41,773 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:41,775 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:41,775 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/4dc99060e5a0416d9735adedd47c40df is 50, key is test_row_0/C:col10/1733344120713/Put/seqid=0 2024-12-04T20:28:41,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d655e27423904899992402a7bb2b35c2 2024-12-04T20:28:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742087_1263 (size=12104) 2024-12-04T20:28:41,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9ab6d367fc19437797d66aa97f38ca3b is 50, key is test_row_0/B:col10/1733344121372/Put/seqid=0 2024-12-04T20:28:41,799 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/4dc99060e5a0416d9735adedd47c40df as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4dc99060e5a0416d9735adedd47c40df 2024-12-04T20:28:41,803 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 4dc99060e5a0416d9735adedd47c40df(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:41,803 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:41,803 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344121313; duration=0sec 2024-12-04T20:28:41,804 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:41,804 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:41,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742088_1264 (size=12001) 2024-12-04T20:28:41,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:41,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:41,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:41,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:41,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344181931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,934 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 2687aa8f5064ce602682f5c7138994be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T20:28:41,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344181951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:41,953 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 2687aa8f5064ce602682f5c7138994be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:42,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344182021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344182021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:42,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344182021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:42,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:42,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:42,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9ab6d367fc19437797d66aa97f38ca3b 2024-12-04T20:28:42,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b5b69e5439b3456eb7cee2bea715141e is 50, key is test_row_0/C:col10/1733344121372/Put/seqid=0 2024-12-04T20:28:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742089_1265 (size=12001) 2024-12-04T20:28:42,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:42,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:42,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344182531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344182531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:42,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344182531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:42,628 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:28:42,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b5b69e5439b3456eb7cee2bea715141e 2024-12-04T20:28:42,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:42,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d655e27423904899992402a7bb2b35c2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d655e27423904899992402a7bb2b35c2 2024-12-04T20:28:42,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d655e27423904899992402a7bb2b35c2, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T20:28:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9ab6d367fc19437797d66aa97f38ca3b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9ab6d367fc19437797d66aa97f38ca3b 2024-12-04T20:28:42,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9ab6d367fc19437797d66aa97f38ca3b, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T20:28:42,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b5b69e5439b3456eb7cee2bea715141e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b5b69e5439b3456eb7cee2bea715141e 2024-12-04T20:28:42,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b5b69e5439b3456eb7cee2bea715141e, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T20:28:42,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2687aa8f5064ce602682f5c7138994be in 1310ms, sequenceid=80, compaction requested=false 2024-12-04T20:28:42,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:42,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T20:28:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:42,812 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:28:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:42,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/286afda85975430b9c82695992a51b55 is 50, key is test_row_0/A:col10/1733344121385/Put/seqid=0 2024-12-04T20:28:42,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742090_1266 (size=12001) 2024-12-04T20:28:42,851 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/286afda85975430b9c82695992a51b55 2024-12-04T20:28:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b4daece59ce641d384df3aeafa306692 is 50, key is test_row_0/B:col10/1733344121385/Put/seqid=0 2024-12-04T20:28:42,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742091_1267 (size=12001) 2024-12-04T20:28:42,867 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b4daece59ce641d384df3aeafa306692 2024-12-04T20:28:42,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/1aa7a2b9808e4049bee093830b54345f is 50, key is test_row_0/C:col10/1733344121385/Put/seqid=0 2024-12-04T20:28:42,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742092_1268 (size=12001) 2024-12-04T20:28:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T20:28:43,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/1aa7a2b9808e4049bee093830b54345f 2024-12-04T20:28:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/286afda85975430b9c82695992a51b55 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/286afda85975430b9c82695992a51b55 2024-12-04T20:28:43,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/286afda85975430b9c82695992a51b55, entries=150, sequenceid=96, filesize=11.7 K 2024-12-04T20:28:43,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b4daece59ce641d384df3aeafa306692 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b4daece59ce641d384df3aeafa306692 2024-12-04T20:28:43,315 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b4daece59ce641d384df3aeafa306692, entries=150, sequenceid=96, filesize=11.7 K 2024-12-04T20:28:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/1aa7a2b9808e4049bee093830b54345f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1aa7a2b9808e4049bee093830b54345f 2024-12-04T20:28:43,321 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1aa7a2b9808e4049bee093830b54345f, entries=150, sequenceid=96, filesize=11.7 K 2024-12-04T20:28:43,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 2687aa8f5064ce602682f5c7138994be in 510ms, sequenceid=96, compaction requested=true 2024-12-04T20:28:43,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:43,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:43,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-04T20:28:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-04T20:28:43,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-04T20:28:43,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5080 sec 2024-12-04T20:28:43,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.5150 sec 2024-12-04T20:28:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:43,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:28:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:43,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/cca8c0d863a646548c343a20b191bb32 is 50, key is test_row_0/A:col10/1733344123547/Put/seqid=0 2024-12-04T20:28:43,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742093_1269 (size=14341) 2024-12-04T20:28:43,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344183618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344123603, max timestamp=1733344123623Num uniques 859; 2024-12-04T20:28:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344123603, max timestamp=1733344123623Num uniques 859; 2024-12-04T20:28:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344123603, max timestamp=1733344123623Num uniques 859; 2024-12-04T20:28:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344183674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344183693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344183731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344183801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344183801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344183941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344183941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,945 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 2687aa8f5064ce602682f5c7138994be of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344183971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:43,973 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 2687aa8f5064ce602682f5c7138994be of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:43,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/cca8c0d863a646548c343a20b191bb32 2024-12-04T20:28:43,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/74430663e8f34676bd945ca5b3fa0478 is 50, key is test_row_0/B:col10/1733344123547/Put/seqid=0 2024-12-04T20:28:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742094_1270 (size=12001) 2024-12-04T20:28:44,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344184012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344184012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344184251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344184321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344184321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/74430663e8f34676bd945ca5b3fa0478 2024-12-04T20:28:44,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/770d7b09190d4238aa273cff6ed93592 is 50, key is test_row_0/C:col10/1733344123547/Put/seqid=0 2024-12-04T20:28:44,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742095_1271 (size=12001) 2024-12-04T20:28:44,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344184761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/770d7b09190d4238aa273cff6ed93592 2024-12-04T20:28:44,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/cca8c0d863a646548c343a20b191bb32 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/cca8c0d863a646548c343a20b191bb32 2024-12-04T20:28:44,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/cca8c0d863a646548c343a20b191bb32, entries=200, sequenceid=107, filesize=14.0 K 2024-12-04T20:28:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/74430663e8f34676bd945ca5b3fa0478 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/74430663e8f34676bd945ca5b3fa0478 2024-12-04T20:28:44,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/74430663e8f34676bd945ca5b3fa0478, entries=150, sequenceid=107, filesize=11.7 K 2024-12-04T20:28:44,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/770d7b09190d4238aa273cff6ed93592 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/770d7b09190d4238aa273cff6ed93592 2024-12-04T20:28:44,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/770d7b09190d4238aa273cff6ed93592, entries=150, sequenceid=107, filesize=11.7 K 2024-12-04T20:28:44,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2687aa8f5064ce602682f5c7138994be in 1282ms, sequenceid=107, compaction requested=true 2024-12-04T20:28:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:44,833 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:44,833 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:44,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:44,835 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:44,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:44,835 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:44,835 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:44,835 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:44,835 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21adbc200940479f90fbb20dfcb4e7bd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d655e27423904899992402a7bb2b35c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/286afda85975430b9c82695992a51b55, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/cca8c0d863a646548c343a20b191bb32] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=49.3 K 2024-12-04T20:28:44,835 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/6d96d2fbfacb4788af30cf9c91d8a1a0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9ab6d367fc19437797d66aa97f38ca3b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b4daece59ce641d384df3aeafa306692, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/74430663e8f34676bd945ca5b3fa0478] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=47.0 K 2024-12-04T20:28:44,836 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d96d2fbfacb4788af30cf9c91d8a1a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344120079 2024-12-04T20:28:44,836 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21adbc200940479f90fbb20dfcb4e7bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344120079 2024-12-04T20:28:44,836 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ab6d367fc19437797d66aa97f38ca3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733344120728 2024-12-04T20:28:44,836 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d655e27423904899992402a7bb2b35c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733344120728 2024-12-04T20:28:44,837 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b4daece59ce641d384df3aeafa306692, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344121385 2024-12-04T20:28:44,837 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 286afda85975430b9c82695992a51b55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344121385 2024-12-04T20:28:44,837 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 74430663e8f34676bd945ca5b3fa0478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733344123546 2024-12-04T20:28:44,837 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting cca8c0d863a646548c343a20b191bb32, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733344123546 2024-12-04T20:28:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:44,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T20:28:44,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344123667, max timestamp=1733344124837Num uniques 1018; 2024-12-04T20:28:44,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:28:44,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344123667, max timestamp=1733344124837Num uniques 1018; 2024-12-04T20:28:44,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:28:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:44,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:44,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:44,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:28:44,856 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:44,857 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/b05a7d3ecb3a4d118874566611fadbf2 is 50, key is test_row_0/A:col10/1733344123547/Put/seqid=0 2024-12-04T20:28:44,867 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:44,868 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/aa5a9f8f6bfb4ca3b25773aaf4eac863 is 50, key is test_row_0/B:col10/1733344123547/Put/seqid=0 2024-12-04T20:28:44,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5979f604388b4a14b18053d0f6661e67 is 50, key is test_row_0/A:col10/1733344124837/Put/seqid=0 2024-12-04T20:28:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344184900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344184904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:44,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742096_1272 (size=12241) 2024-12-04T20:28:44,958 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/b05a7d3ecb3a4d118874566611fadbf2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b05a7d3ecb3a4d118874566611fadbf2 2024-12-04T20:28:44,965 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into b05a7d3ecb3a4d118874566611fadbf2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:44,966 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:44,966 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=12, startTime=1733344124833; duration=0sec 2024-12-04T20:28:44,966 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:44,966 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:44,966 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:28:44,968 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:28:44,968 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:44,968 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:44,969 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4dc99060e5a0416d9735adedd47c40df, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b5b69e5439b3456eb7cee2bea715141e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1aa7a2b9808e4049bee093830b54345f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/770d7b09190d4238aa273cff6ed93592] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=47.0 K 2024-12-04T20:28:44,969 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc99060e5a0416d9735adedd47c40df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733344120079 2024-12-04T20:28:44,969 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5b69e5439b3456eb7cee2bea715141e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733344120728 2024-12-04T20:28:44,970 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa7a2b9808e4049bee093830b54345f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344121385 2024-12-04T20:28:44,970 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 770d7b09190d4238aa273cff6ed93592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733344123546 2024-12-04T20:28:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T20:28:44,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742097_1273 (size=12241) 2024-12-04T20:28:44,984 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:44,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-04T20:28:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T20:28:44,988 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:44,989 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:44,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:44,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742098_1274 (size=12151) 2024-12-04T20:28:44,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5979f604388b4a14b18053d0f6661e67 2024-12-04T20:28:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344185011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:45,026 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#225 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:45,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/606cf469f3ed478cbab3abf457de807d is 50, key is test_row_0/B:col10/1733344124837/Put/seqid=0 2024-12-04T20:28:45,028 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/6aea62bee8c9465ab58f3dcfd0afbe81 is 50, key is test_row_0/C:col10/1733344123547/Put/seqid=0 2024-12-04T20:28:45,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344185022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:45,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742099_1275 (size=12241) 2024-12-04T20:28:45,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742100_1276 (size=12151) 2024-12-04T20:28:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T20:28:45,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344185221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:45,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344185242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:45,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T20:28:45,387 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/aa5a9f8f6bfb4ca3b25773aaf4eac863 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/aa5a9f8f6bfb4ca3b25773aaf4eac863 2024-12-04T20:28:45,391 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into aa5a9f8f6bfb4ca3b25773aaf4eac863(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:45,391 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:45,391 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=12, startTime=1733344124833; duration=0sec 2024-12-04T20:28:45,392 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:45,392 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:45,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:45,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,459 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/6aea62bee8c9465ab58f3dcfd0afbe81 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6aea62bee8c9465ab58f3dcfd0afbe81 2024-12-04T20:28:45,465 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 6aea62bee8c9465ab58f3dcfd0afbe81(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:45,465 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:45,465 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=12, startTime=1733344124833; duration=0sec 2024-12-04T20:28:45,465 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:45,465 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:45,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/606cf469f3ed478cbab3abf457de807d 2024-12-04T20:28:45,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/8f87f07ca05b460eb04f01248b57a61e is 50, key is test_row_0/C:col10/1733344124837/Put/seqid=0 2024-12-04T20:28:45,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742101_1277 (size=12151) 2024-12-04T20:28:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344185531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:45,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344185551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:45,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T20:28:45,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344185772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:45,776 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 2687aa8f5064ce602682f5c7138994be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:28:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:45,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/8f87f07ca05b460eb04f01248b57a61e 2024-12-04T20:28:45,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5979f604388b4a14b18053d0f6661e67 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5979f604388b4a14b18053d0f6661e67 2024-12-04T20:28:45,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:45,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5979f604388b4a14b18053d0f6661e67, entries=150, sequenceid=145, filesize=11.9 K 2024-12-04T20:28:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:45,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/606cf469f3ed478cbab3abf457de807d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/606cf469f3ed478cbab3abf457de807d 2024-12-04T20:28:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:45,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/606cf469f3ed478cbab3abf457de807d, entries=150, sequenceid=145, filesize=11.9 K 2024-12-04T20:28:45,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/8f87f07ca05b460eb04f01248b57a61e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8f87f07ca05b460eb04f01248b57a61e 2024-12-04T20:28:45,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8f87f07ca05b460eb04f01248b57a61e, entries=150, sequenceid=145, filesize=11.9 K 2024-12-04T20:28:45,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=114.05 KB/116790 for 2687aa8f5064ce602682f5c7138994be in 1082ms, sequenceid=145, compaction requested=false 2024-12-04T20:28:45,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:46,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T20:28:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:46,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/72299a5e72024dbc8039efe2d4c00b3e is 50, key is test_row_0/A:col10/1733344124901/Put/seqid=0 2024-12-04T20:28:46,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742102_1278 (size=12151) 2024-12-04T20:28:46,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/72299a5e72024dbc8039efe2d4c00b3e 2024-12-04T20:28:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344186082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344186082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/0bc9f703d90d43f0bbc4b8cb167c0c54 is 50, key is test_row_0/B:col10/1733344124901/Put/seqid=0 2024-12-04T20:28:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T20:28:46,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742103_1279 (size=12151) 2024-12-04T20:28:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344186192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344186192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:46,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:46,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:46,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:46,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344186402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344186402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:46,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/0bc9f703d90d43f0bbc4b8cb167c0c54 2024-12-04T20:28:46,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/8b5b4802f93440b683a6b3d0cc0b4d24 is 50, key is test_row_0/C:col10/1733344124901/Put/seqid=0 2024-12-04T20:28:46,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742104_1280 (size=12151) 2024-12-04T20:28:46,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/8b5b4802f93440b683a6b3d0cc0b4d24 2024-12-04T20:28:46,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/72299a5e72024dbc8039efe2d4c00b3e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/72299a5e72024dbc8039efe2d4c00b3e 2024-12-04T20:28:46,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/72299a5e72024dbc8039efe2d4c00b3e, entries=150, sequenceid=169, filesize=11.9 K 2024-12-04T20:28:46,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/0bc9f703d90d43f0bbc4b8cb167c0c54 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0bc9f703d90d43f0bbc4b8cb167c0c54 2024-12-04T20:28:46,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0bc9f703d90d43f0bbc4b8cb167c0c54, entries=150, sequenceid=169, filesize=11.9 K 2024-12-04T20:28:46,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/8b5b4802f93440b683a6b3d0cc0b4d24 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8b5b4802f93440b683a6b3d0cc0b4d24 2024-12-04T20:28:46,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8b5b4802f93440b683a6b3d0cc0b4d24, entries=150, sequenceid=169, filesize=11.9 K 2024-12-04T20:28:46,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 2687aa8f5064ce602682f5c7138994be in 516ms, sequenceid=169, compaction requested=true 2024-12-04T20:28:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:46,561 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T20:28:46,562 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:46,563 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:46,563 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:46,564 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:46,564 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:46,564 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,564 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/aa5a9f8f6bfb4ca3b25773aaf4eac863, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/606cf469f3ed478cbab3abf457de807d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0bc9f703d90d43f0bbc4b8cb167c0c54] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=35.7 K 2024-12-04T20:28:46,564 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,564 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b05a7d3ecb3a4d118874566611fadbf2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5979f604388b4a14b18053d0f6661e67, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/72299a5e72024dbc8039efe2d4c00b3e] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=35.7 K 2024-12-04T20:28:46,564 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b05a7d3ecb3a4d118874566611fadbf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733344123546 2024-12-04T20:28:46,564 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa5a9f8f6bfb4ca3b25773aaf4eac863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733344123546 2024-12-04T20:28:46,565 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 5979f604388b4a14b18053d0f6661e67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733344123667 2024-12-04T20:28:46,565 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 606cf469f3ed478cbab3abf457de807d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733344123667 2024-12-04T20:28:46,565 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 72299a5e72024dbc8039efe2d4c00b3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733344124891 2024-12-04T20:28:46,565 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc9f703d90d43f0bbc4b8cb167c0c54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733344124891 2024-12-04T20:28:46,580 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:46,580 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/48f86036eaa94e4c8b3c01a1e81b7fea is 50, key is test_row_0/A:col10/1733344124901/Put/seqid=0 2024-12-04T20:28:46,623 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:46,624 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/1c601ae6dfb849a5aca5c0ac3fdf32ad is 50, key is test_row_0/B:col10/1733344124901/Put/seqid=0 2024-12-04T20:28:46,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742105_1281 (size=12493) 2024-12-04T20:28:46,669 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/48f86036eaa94e4c8b3c01a1e81b7fea as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/48f86036eaa94e4c8b3c01a1e81b7fea 2024-12-04T20:28:46,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T20:28:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,673 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:28:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:46,675 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into 48f86036eaa94e4c8b3c01a1e81b7fea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:46,675 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:46,675 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344126561; duration=0sec 2024-12-04T20:28:46,675 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:46,675 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:46,675 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:46,677 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:46,677 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:46,677 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:46,678 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6aea62bee8c9465ab58f3dcfd0afbe81, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8f87f07ca05b460eb04f01248b57a61e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8b5b4802f93440b683a6b3d0cc0b4d24] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=35.7 K 2024-12-04T20:28:46,679 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aea62bee8c9465ab58f3dcfd0afbe81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733344123546 2024-12-04T20:28:46,679 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f87f07ca05b460eb04f01248b57a61e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733344123667 2024-12-04T20:28:46,679 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b5b4802f93440b683a6b3d0cc0b4d24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733344124891 2024-12-04T20:28:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/43d26d25ca0647b5a866fee6d780cb79 is 50, key is test_row_0/A:col10/1733344126079/Put/seqid=0 2024-12-04T20:28:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:46,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742106_1282 (size=12493) 2024-12-04T20:28:46,723 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:46,723 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/22be1721d093445eb6876aef78a723ca is 50, key is test_row_0/C:col10/1733344124901/Put/seqid=0 2024-12-04T20:28:46,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742108_1284 (size=12493) 2024-12-04T20:28:46,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742107_1283 (size=12151) 2024-12-04T20:28:46,756 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/43d26d25ca0647b5a866fee6d780cb79 2024-12-04T20:28:46,764 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/22be1721d093445eb6876aef78a723ca as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/22be1721d093445eb6876aef78a723ca 2024-12-04T20:28:46,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344186760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344186762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/5c363af1791242a79c6a755932da0d1e is 50, key is test_row_0/B:col10/1733344126079/Put/seqid=0 2024-12-04T20:28:46,776 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 22be1721d093445eb6876aef78a723ca(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:46,776 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:46,776 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344126561; duration=0sec 2024-12-04T20:28:46,776 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:46,776 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:46,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742109_1285 (size=12151) 2024-12-04T20:28:46,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344186871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344186871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344187082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:47,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344187082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:47,126 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/1c601ae6dfb849a5aca5c0ac3fdf32ad as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1c601ae6dfb849a5aca5c0ac3fdf32ad 2024-12-04T20:28:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T20:28:47,142 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into 1c601ae6dfb849a5aca5c0ac3fdf32ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:47,142 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:47,142 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344126561; duration=0sec 2024-12-04T20:28:47,142 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:47,142 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:47,192 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/5c363af1791242a79c6a755932da0d1e 2024-12-04T20:28:47,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/cdb2d6d940264ca6abd0d9f19587b9e9 is 50, key is test_row_0/C:col10/1733344126079/Put/seqid=0 2024-12-04T20:28:47,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742110_1286 (size=12151) 2024-12-04T20:28:47,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344187396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:47,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344187396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:47,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/cdb2d6d940264ca6abd0d9f19587b9e9 2024-12-04T20:28:47,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/43d26d25ca0647b5a866fee6d780cb79 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/43d26d25ca0647b5a866fee6d780cb79 2024-12-04T20:28:47,657 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/43d26d25ca0647b5a866fee6d780cb79, entries=150, sequenceid=181, filesize=11.9 K 2024-12-04T20:28:47,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/5c363af1791242a79c6a755932da0d1e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5c363af1791242a79c6a755932da0d1e 2024-12-04T20:28:47,663 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5c363af1791242a79c6a755932da0d1e, entries=150, sequenceid=181, filesize=11.9 K 2024-12-04T20:28:47,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/cdb2d6d940264ca6abd0d9f19587b9e9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/cdb2d6d940264ca6abd0d9f19587b9e9 2024-12-04T20:28:47,669 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/cdb2d6d940264ca6abd0d9f19587b9e9, entries=150, sequenceid=181, filesize=11.9 K 2024-12-04T20:28:47,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2687aa8f5064ce602682f5c7138994be in 997ms, sequenceid=181, compaction requested=false 2024-12-04T20:28:47,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:47,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:47,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-04T20:28:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-04T20:28:47,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-04T20:28:47,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6830 sec 2024-12-04T20:28:47,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6940 sec 2024-12-04T20:28:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:47,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344126727, max timestamp=1733344127803Num uniques 859; 2024-12-04T20:28:47,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344126727, max timestamp=1733344127803Num uniques 859; 2024-12-04T20:28:47,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:47,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:47,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-04T20:28:47,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344126727, max timestamp=1733344127803Num uniques 859; 2024-12-04T20:28:47,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:47,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6da640fb0fc54d829d8375b27e034571 is 50, key is test_row_0/A:col10/1733344126760/Put/seqid=0 2024-12-04T20:28:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742111_1287 (size=14541) 2024-12-04T20:28:47,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6da640fb0fc54d829d8375b27e034571 2024-12-04T20:28:47,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/3706932d91e4454181fa778178a1a191 is 50, key is test_row_0/B:col10/1733344126760/Put/seqid=0 2024-12-04T20:28:47,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742112_1288 (size=12151) 2024-12-04T20:28:47,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/3706932d91e4454181fa778178a1a191 2024-12-04T20:28:47,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/e5f2dfb6d2c743c4aa060626c7688ea6 is 50, key is test_row_0/C:col10/1733344126760/Put/seqid=0 2024-12-04T20:28:47,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742113_1289 (size=12151) 2024-12-04T20:28:47,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344127861, max timestamp=1733344127893Num uniques 859; 2024-12-04T20:28:47,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344127861, max timestamp=1733344127893Num uniques 859; 2024-12-04T20:28:47,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:47,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:47,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344127861, max timestamp=1733344127893Num uniques 859; 2024-12-04T20:28:47,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344127924, max timestamp=1733344127935Num uniques 1018; 2024-12-04T20:28:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:28:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344127924, max timestamp=1733344127935Num uniques 1018; 2024-12-04T20:28:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:28:47,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733344127924, max timestamp=1733344127935Num uniques 1018; 2024-12-04T20:28:47,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-04T20:28:47,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733344127972, max timestamp=1733344127991Num uniques 1009; 2024-12-04T20:28:47,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-12-04T20:28:47,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733344127972, max timestamp=1733344127991Num uniques 1027; 2024-12-04T20:28:47,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T20:28:47,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733344127972, max timestamp=1733344127991Num uniques 1027; 2024-12-04T20:28:47,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T20:28:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733344128003, max timestamp=1733344128015Num uniques 1086; 2024-12-04T20:28:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733344128003, max timestamp=1733344128015Num uniques 1086; 2024-12-04T20:28:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=8506, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-04T20:28:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=8506, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-04T20:28:48,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733344128003, max timestamp=1733344128015Num uniques 1086; 2024-12-04T20:28:48,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=8506, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-04T20:28:48,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733344128035, max timestamp=1733344128043Num uniques 1045; 2024-12-04T20:28:48,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T20:28:48,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733344128035, max timestamp=1733344128043Num uniques 1045; 2024-12-04T20:28:48,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T20:28:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733344128035, max timestamp=1733344128043Num uniques 1045; 2024-12-04T20:28:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T20:28:48,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733344128052, max timestamp=1733344128069Num uniques 1104; 2024-12-04T20:28:48,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-12-04T20:28:48,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733344128052, max timestamp=1733344128069Num uniques 1104; 2024-12-04T20:28:48,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-12-04T20:28:48,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733344128062, max timestamp=1733344128069Num uniques 1104; 2024-12-04T20:28:48,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-04T20:28:48,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733344128085, max timestamp=1733344128097Num uniques 1163; 2024-12-04T20:28:48,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=12038, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-04T20:28:48,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733344128085, max timestamp=1733344128097Num uniques 1163; 2024-12-04T20:28:48,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=12038, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-04T20:28:48,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733344128091, max timestamp=1733344128097Num uniques 1063; 2024-12-04T20:28:48,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-04T20:28:48,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733344128119, max timestamp=1733344128127Num uniques 1122; 2024-12-04T20:28:48,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733344128119, max timestamp=1733344128127Num uniques 1022; 2024-12-04T20:28:48,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-04T20:28:48,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-04T20:28:48,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733344128119, max timestamp=1733344128127Num uniques 1122; 2024-12-04T20:28:48,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-04T20:28:48,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733344128153, max timestamp=1733344128160Num uniques 1031; 2024-12-04T20:28:48,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-04T20:28:48,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733344128153, max timestamp=1733344128160Num uniques 1031; 2024-12-04T20:28:48,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733344128153, max timestamp=1733344128160Num uniques 1031; 2024-12-04T20:28:48,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-04T20:28:48,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-04T20:28:48,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733344128184, max timestamp=1733344128196Num uniques 1040; 2024-12-04T20:28:48,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-04T20:28:48,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733344128184, max timestamp=1733344128196Num uniques 1040; 2024-12-04T20:28:48,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-04T20:28:48,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733344128184, max timestamp=1733344128196Num uniques 1040; 2024-12-04T20:28:48,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-04T20:28:48,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733344128218, max timestamp=1733344128227Num uniques 1049; 2024-12-04T20:28:48,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-04T20:28:48,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733344128218, max timestamp=1733344128227Num uniques 1049; 2024-12-04T20:28:48,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-04T20:28:48,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733344128218, max timestamp=1733344128227Num uniques 1049; 2024-12-04T20:28:48,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-04T20:28:48,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733344128252, max timestamp=1733344128264Num uniques 1108; 2024-12-04T20:28:48,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-04T20:28:48,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733344128252, max timestamp=1733344128264Num uniques 1108; 2024-12-04T20:28:48,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-04T20:28:48,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733344128252, max timestamp=1733344128264Num uniques 1108; 2024-12-04T20:28:48,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-04T20:28:48,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/e5f2dfb6d2c743c4aa060626c7688ea6 2024-12-04T20:28:48,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6da640fb0fc54d829d8375b27e034571 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6da640fb0fc54d829d8375b27e034571 2024-12-04T20:28:48,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6da640fb0fc54d829d8375b27e034571, entries=200, sequenceid=205, filesize=14.2 K 2024-12-04T20:28:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/3706932d91e4454181fa778178a1a191 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3706932d91e4454181fa778178a1a191 2024-12-04T20:28:48,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3706932d91e4454181fa778178a1a191, entries=150, sequenceid=205, filesize=11.9 K 2024-12-04T20:28:48,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/e5f2dfb6d2c743c4aa060626c7688ea6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e5f2dfb6d2c743c4aa060626c7688ea6 2024-12-04T20:28:48,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733344128287, max timestamp=1733344128302Num uniques 1117; 2024-12-04T20:28:48,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=12222, suffix heap size=144136,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=267 2024-12-04T20:28:48,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733344128287, max timestamp=1733344128302Num uniques 1117; 2024-12-04T20:28:48,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=12222, suffix heap size=144136,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=267 2024-12-04T20:28:48,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e5f2dfb6d2c743c4aa060626c7688ea6, entries=150, sequenceid=205, filesize=11.9 K 2024-12-04T20:28:48,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733344128287, max timestamp=1733344128302Num uniques 1117; 2024-12-04T20:28:48,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=12222, suffix heap size=144136,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=267 2024-12-04T20:28:48,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=40.25 KB/41220 for 2687aa8f5064ce602682f5c7138994be in 503ms, sequenceid=205, compaction requested=true 2024-12-04T20:28:48,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:48,309 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:48,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:48,310 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:48,310 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:48,310 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:48,310 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/48f86036eaa94e4c8b3c01a1e81b7fea, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/43d26d25ca0647b5a866fee6d780cb79, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6da640fb0fc54d829d8375b27e034571] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=38.3 K 2024-12-04T20:28:48,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:48,310 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:48,311 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f86036eaa94e4c8b3c01a1e81b7fea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733344124891 2024-12-04T20:28:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:48,312 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43d26d25ca0647b5a866fee6d780cb79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733344126072 2024-12-04T20:28:48,312 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:48,312 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:48,312 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:48,312 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1c601ae6dfb849a5aca5c0ac3fdf32ad, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5c363af1791242a79c6a755932da0d1e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3706932d91e4454181fa778178a1a191] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=35.9 K 2024-12-04T20:28:48,312 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c601ae6dfb849a5aca5c0ac3fdf32ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733344124891 2024-12-04T20:28:48,313 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da640fb0fc54d829d8375b27e034571, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344126727 2024-12-04T20:28:48,313 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c363af1791242a79c6a755932da0d1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733344126072 2024-12-04T20:28:48,314 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3706932d91e4454181fa778178a1a191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344126753 2024-12-04T20:28:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:48,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=80.51 KB heapSize=215.48 KB 2024-12-04T20:28:48,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:48,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:48,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:48,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:48,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:48,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:48,331 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:48,331 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b2586ffa58f44b0a9ba1ff9bc5316866 is 50, key is test_row_0/B:col10/1733344126760/Put/seqid=0 2024-12-04T20:28:48,334 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:48,334 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/b47c1d49f2ec4309b76b6ea8e3a6a1b6 is 50, key is test_row_0/A:col10/1733344126760/Put/seqid=0 2024-12-04T20:28:48,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/f146241a55a44ea7ba5e78bbaf76657b is 50, key is test_row_0/A:col10/1733344128306/Put/seqid=0 2024-12-04T20:28:48,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344188340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344188339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344188342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344188347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344188347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742115_1291 (size=17181) 2024-12-04T20:28:48,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742116_1292 (size=12595) 2024-12-04T20:28:48,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742114_1290 (size=12595) 2024-12-04T20:28:48,385 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/b47c1d49f2ec4309b76b6ea8e3a6a1b6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b47c1d49f2ec4309b76b6ea8e3a6a1b6 2024-12-04T20:28:48,392 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into b47c1d49f2ec4309b76b6ea8e3a6a1b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:48,392 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:48,392 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344128309; duration=0sec 2024-12-04T20:28:48,392 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:48,392 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:48,392 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:48,394 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:48,394 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:48,394 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:48,394 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/22be1721d093445eb6876aef78a723ca, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/cdb2d6d940264ca6abd0d9f19587b9e9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e5f2dfb6d2c743c4aa060626c7688ea6] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=35.9 K 2024-12-04T20:28:48,394 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22be1721d093445eb6876aef78a723ca, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733344124891 2024-12-04T20:28:48,395 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb2d6d940264ca6abd0d9f19587b9e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733344126072 2024-12-04T20:28:48,395 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f2dfb6d2c743c4aa060626c7688ea6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344126753 2024-12-04T20:28:48,403 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:48,403 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/1cd65ee603b340b2bff0be141cde937f is 50, key is test_row_0/C:col10/1733344126760/Put/seqid=0 2024-12-04T20:28:48,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742117_1293 (size=12595) 2024-12-04T20:28:48,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344188451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344188451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344188452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344188462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344188462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344188661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344188661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344188661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344188672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344188672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/f146241a55a44ea7ba5e78bbaf76657b 2024-12-04T20:28:48,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/3b3a6e2ce5ad4867b18373dc1a7f83ae is 50, key is test_row_0/B:col10/1733344128306/Put/seqid=0 2024-12-04T20:28:48,779 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b2586ffa58f44b0a9ba1ff9bc5316866 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b2586ffa58f44b0a9ba1ff9bc5316866 2024-12-04T20:28:48,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742118_1294 (size=12301) 2024-12-04T20:28:48,787 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into b2586ffa58f44b0a9ba1ff9bc5316866(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:48,787 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:48,787 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344128310; duration=0sec 2024-12-04T20:28:48,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/3b3a6e2ce5ad4867b18373dc1a7f83ae 2024-12-04T20:28:48,787 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:48,787 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:48,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b50c48bf3ef8491ca0f2930dc420d3ec is 50, key is test_row_0/C:col10/1733344128306/Put/seqid=0 2024-12-04T20:28:48,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742119_1295 (size=12301) 2024-12-04T20:28:48,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b50c48bf3ef8491ca0f2930dc420d3ec 2024-12-04T20:28:48,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/f146241a55a44ea7ba5e78bbaf76657b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/f146241a55a44ea7ba5e78bbaf76657b 2024-12-04T20:28:48,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/f146241a55a44ea7ba5e78bbaf76657b, entries=250, sequenceid=439, filesize=16.8 K 2024-12-04T20:28:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/3b3a6e2ce5ad4867b18373dc1a7f83ae as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3b3a6e2ce5ad4867b18373dc1a7f83ae 2024-12-04T20:28:48,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3b3a6e2ce5ad4867b18373dc1a7f83ae, entries=150, sequenceid=439, filesize=12.0 K 2024-12-04T20:28:48,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b50c48bf3ef8491ca0f2930dc420d3ec as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b50c48bf3ef8491ca0f2930dc420d3ec 2024-12-04T20:28:48,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b50c48bf3ef8491ca0f2930dc420d3ec, entries=150, sequenceid=439, filesize=12.0 K 2024-12-04T20:28:48,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~235.36 KB/241008, currentSize=100.63 KB/103050 for 2687aa8f5064ce602682f5c7138994be in 509ms, sequenceid=439, compaction requested=false 2024-12-04T20:28:48,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:48,840 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/1cd65ee603b340b2bff0be141cde937f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1cd65ee603b340b2bff0be141cde937f 2024-12-04T20:28:48,846 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 1cd65ee603b340b2bff0be141cde937f(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:48,846 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:48,846 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344128317; duration=0sec 2024-12-04T20:28:48,846 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:48,846 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:48,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T20:28:48,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:48,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:48,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:48,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:48,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:48,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:48,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/b8475403f80f4280ad9cd95bec43509d is 50, key is test_row_0/A:col10/1733344128345/Put/seqid=0 2024-12-04T20:28:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742120_1296 (size=12301) 2024-12-04T20:28:48,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/b8475403f80f4280ad9cd95bec43509d 2024-12-04T20:28:48,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344188985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344188986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344188987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/a056440075514294a4fd6aa959b59dc2 is 50, key is test_row_0/B:col10/1733344128345/Put/seqid=0 2024-12-04T20:28:48,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344188989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344188989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:48,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742121_1297 (size=12301) 2024-12-04T20:28:49,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344189102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344189102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344189102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344189102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344189102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T20:28:49,152 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:49,152 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:49,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-04T20:28:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T20:28:49,155 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:49,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:49,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T20:28:49,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T20:28:49,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:49,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:49,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:49,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:49,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344189312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344189312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344189314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344189312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344189314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/a056440075514294a4fd6aa959b59dc2 2024-12-04T20:28:49,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/6ec0e7b60c9b45ec978e0530af355b36 is 50, key is test_row_0/C:col10/1733344128345/Put/seqid=0 2024-12-04T20:28:49,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742122_1298 (size=12301) 2024-12-04T20:28:49,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/6ec0e7b60c9b45ec978e0530af355b36 2024-12-04T20:28:49,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/b8475403f80f4280ad9cd95bec43509d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b8475403f80f4280ad9cd95bec43509d 2024-12-04T20:28:49,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b8475403f80f4280ad9cd95bec43509d, entries=150, sequenceid=461, filesize=12.0 K 2024-12-04T20:28:49,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/a056440075514294a4fd6aa959b59dc2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/a056440075514294a4fd6aa959b59dc2 2024-12-04T20:28:49,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/a056440075514294a4fd6aa959b59dc2, entries=150, sequenceid=461, filesize=12.0 K 2024-12-04T20:28:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/6ec0e7b60c9b45ec978e0530af355b36 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6ec0e7b60c9b45ec978e0530af355b36 2024-12-04T20:28:49,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6ec0e7b60c9b45ec978e0530af355b36, entries=150, sequenceid=461, filesize=12.0 K 2024-12-04T20:28:49,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 2687aa8f5064ce602682f5c7138994be in 487ms, sequenceid=461, compaction requested=true 2024-12-04T20:28:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:49,461 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:49,461 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:49,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:49,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:49,462 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:49,462 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:49,462 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:49,463 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b47c1d49f2ec4309b76b6ea8e3a6a1b6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/f146241a55a44ea7ba5e78bbaf76657b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b8475403f80f4280ad9cd95bec43509d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=41.1 K 2024-12-04T20:28:49,463 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:49,463 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:49,463 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:49,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T20:28:49,463 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b2586ffa58f44b0a9ba1ff9bc5316866, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3b3a6e2ce5ad4867b18373dc1a7f83ae, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/a056440075514294a4fd6aa959b59dc2] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.3 K 2024-12-04T20:28:49,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:49,463 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T20:28:49,463 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting b47c1d49f2ec4309b76b6ea8e3a6a1b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344126753 2024-12-04T20:28:49,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:49,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:49,464 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b2586ffa58f44b0a9ba1ff9bc5316866, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344126753 2024-12-04T20:28:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:49,464 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b3a6e2ce5ad4867b18373dc1a7f83ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733344128306 2024-12-04T20:28:49,464 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f146241a55a44ea7ba5e78bbaf76657b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733344128306 2024-12-04T20:28:49,465 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a056440075514294a4fd6aa959b59dc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733344128339 2024-12-04T20:28:49,465 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8475403f80f4280ad9cd95bec43509d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733344128339 2024-12-04T20:28:49,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6c9bb4a835f24ef1b352e16caae1dfeb is 50, key is test_row_0/A:col10/1733344128984/Put/seqid=0 2024-12-04T20:28:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T20:28:49,489 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#250 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:49,489 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/21824be7213642fbb4fc1bebd0b71add is 50, key is test_row_0/A:col10/1733344128345/Put/seqid=0 2024-12-04T20:28:49,491 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:49,491 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/95ad53e9ae5849c28d407d48778fbca0 is 50, key is test_row_0/B:col10/1733344128345/Put/seqid=0 2024-12-04T20:28:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742123_1299 (size=12301) 2024-12-04T20:28:49,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6c9bb4a835f24ef1b352e16caae1dfeb 2024-12-04T20:28:49,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742124_1300 (size=12847) 2024-12-04T20:28:49,544 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/21824be7213642fbb4fc1bebd0b71add as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21824be7213642fbb4fc1bebd0b71add 2024-12-04T20:28:49,550 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into 21824be7213642fbb4fc1bebd0b71add(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:49,550 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:49,550 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344129461; duration=0sec 2024-12-04T20:28:49,550 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:49,550 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:49,550 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b0584eb66399490e843fad3e2d553456 is 50, key is test_row_0/B:col10/1733344128984/Put/seqid=0 2024-12-04T20:28:49,553 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:49,554 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:49,554 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:49,556 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1cd65ee603b340b2bff0be141cde937f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b50c48bf3ef8491ca0f2930dc420d3ec, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6ec0e7b60c9b45ec978e0530af355b36] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.3 K 2024-12-04T20:28:49,556 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd65ee603b340b2bff0be141cde937f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344126753 2024-12-04T20:28:49,557 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting b50c48bf3ef8491ca0f2930dc420d3ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733344128306 2024-12-04T20:28:49,557 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec0e7b60c9b45ec978e0530af355b36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733344128339 2024-12-04T20:28:49,567 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:49,567 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/6492a7d2ffd8418b850fb9bcd3a56927 is 50, key is test_row_0/C:col10/1733344128345/Put/seqid=0 2024-12-04T20:28:49,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742125_1301 (size=12847) 2024-12-04T20:28:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742126_1302 (size=12301) 2024-12-04T20:28:49,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742127_1303 (size=12847) 2024-12-04T20:28:49,605 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/6492a7d2ffd8418b850fb9bcd3a56927 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6492a7d2ffd8418b850fb9bcd3a56927 2024-12-04T20:28:49,610 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 6492a7d2ffd8418b850fb9bcd3a56927(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:49,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:49,610 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344129462; duration=0sec 2024-12-04T20:28:49,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:49,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:49,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:49,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344189635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344189636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344189637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344189637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344189638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344189751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344189752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344189752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344189752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344189753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T20:28:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344189962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344189962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344189962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344189963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344189969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:49,980 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/95ad53e9ae5849c28d407d48778fbca0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/95ad53e9ae5849c28d407d48778fbca0 2024-12-04T20:28:49,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b0584eb66399490e843fad3e2d553456 2024-12-04T20:28:49,986 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into 95ad53e9ae5849c28d407d48778fbca0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:49,986 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:49,986 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344129461; duration=0sec 2024-12-04T20:28:49,986 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:49,986 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:49,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/0b64794ea87d4472a46027d52fb371fc is 50, key is test_row_0/C:col10/1733344128984/Put/seqid=0 2024-12-04T20:28:50,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742128_1304 (size=12301) 2024-12-04T20:28:50,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/0b64794ea87d4472a46027d52fb371fc 2024-12-04T20:28:50,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6c9bb4a835f24ef1b352e16caae1dfeb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6c9bb4a835f24ef1b352e16caae1dfeb 2024-12-04T20:28:50,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6c9bb4a835f24ef1b352e16caae1dfeb, entries=150, sequenceid=475, filesize=12.0 K 2024-12-04T20:28:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/b0584eb66399490e843fad3e2d553456 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b0584eb66399490e843fad3e2d553456 2024-12-04T20:28:50,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b0584eb66399490e843fad3e2d553456, entries=150, sequenceid=475, filesize=12.0 K 2024-12-04T20:28:50,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/0b64794ea87d4472a46027d52fb371fc as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/0b64794ea87d4472a46027d52fb371fc 2024-12-04T20:28:50,026 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/0b64794ea87d4472a46027d52fb371fc, entries=150, sequenceid=475, filesize=12.0 K 2024-12-04T20:28:50,026 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2687aa8f5064ce602682f5c7138994be in 563ms, sequenceid=475, compaction requested=false 2024-12-04T20:28:50,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:50,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-04T20:28:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-04T20:28:50,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-04T20:28:50,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 872 msec 2024-12-04T20:28:50,031 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 877 msec 2024-12-04T20:28:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:50,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T20:28:50,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:50,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:50,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:50,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:50,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:50,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344190283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344190285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344190285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344190286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344190288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T20:28:50,292 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:50,293 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-04T20:28:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T20:28:50,296 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:50,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:50,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:50,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/8266612e68d94736ac1ee6d1021ca15f is 50, key is test_row_0/A:col10/1733344129636/Put/seqid=0 2024-12-04T20:28:50,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742129_1305 (size=14741) 2024-12-04T20:28:50,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/8266612e68d94736ac1ee6d1021ca15f 2024-12-04T20:28:50,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9b09c683c47e42f19922c753c9bc90d5 is 50, key is test_row_0/B:col10/1733344129636/Put/seqid=0 2024-12-04T20:28:50,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742130_1306 (size=12301) 2024-12-04T20:28:50,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344190391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344190392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344190392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344190392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T20:28:50,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344190401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T20:28:50,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:50,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344190602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344190602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344190602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344190602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T20:28:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T20:28:50,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344190611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9b09c683c47e42f19922c753c9bc90d5 2024-12-04T20:28:50,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/89d78c39d5dc4c04b7d1c3ecafae74c7 is 50, key is test_row_0/C:col10/1733344129636/Put/seqid=0 2024-12-04T20:28:50,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742131_1307 (size=12301) 2024-12-04T20:28:50,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/89d78c39d5dc4c04b7d1c3ecafae74c7 2024-12-04T20:28:50,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/8266612e68d94736ac1ee6d1021ca15f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8266612e68d94736ac1ee6d1021ca15f 2024-12-04T20:28:50,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8266612e68d94736ac1ee6d1021ca15f, entries=200, sequenceid=497, filesize=14.4 K 2024-12-04T20:28:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9b09c683c47e42f19922c753c9bc90d5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9b09c683c47e42f19922c753c9bc90d5 2024-12-04T20:28:50,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9b09c683c47e42f19922c753c9bc90d5, entries=150, sequenceid=497, filesize=12.0 K 2024-12-04T20:28:50,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/89d78c39d5dc4c04b7d1c3ecafae74c7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/89d78c39d5dc4c04b7d1c3ecafae74c7 2024-12-04T20:28:50,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/89d78c39d5dc4c04b7d1c3ecafae74c7, entries=150, sequenceid=497, filesize=12.0 K 2024-12-04T20:28:50,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2687aa8f5064ce602682f5c7138994be in 486ms, sequenceid=497, compaction requested=true 2024-12-04T20:28:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:50,759 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:50,759 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:50,760 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:50,760 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:50,760 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,760 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/95ad53e9ae5849c28d407d48778fbca0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b0584eb66399490e843fad3e2d553456, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9b09c683c47e42f19922c753c9bc90d5] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.6 K 2024-12-04T20:28:50,761 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:50,761 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:50,761 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,761 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21824be7213642fbb4fc1bebd0b71add, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6c9bb4a835f24ef1b352e16caae1dfeb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8266612e68d94736ac1ee6d1021ca15f] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=39.0 K 2024-12-04T20:28:50,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T20:28:50,761 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21824be7213642fbb4fc1bebd0b71add, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733344128339 2024-12-04T20:28:50,761 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ad53e9ae5849c28d407d48778fbca0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733344128339 2024-12-04T20:28:50,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:28:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:50,762 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9bb4a835f24ef1b352e16caae1dfeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733344128983 2024-12-04T20:28:50,762 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b0584eb66399490e843fad3e2d553456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733344128983 2024-12-04T20:28:50,762 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8266612e68d94736ac1ee6d1021ca15f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733344129634 2024-12-04T20:28:50,763 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b09c683c47e42f19922c753c9bc90d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733344129634 2024-12-04T20:28:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/82f1fe18be6c4ef2b229c91ca7853b8f is 50, key is test_row_0/A:col10/1733344130286/Put/seqid=0 2024-12-04T20:28:50,779 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:50,779 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/ccfb902d9fdd4d0c94188575c37f6088 is 50, key is test_row_0/B:col10/1733344129636/Put/seqid=0 2024-12-04T20:28:50,799 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:50,800 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/c4619946d254446a8a133148795f4155 is 50, key is test_row_0/A:col10/1733344129636/Put/seqid=0 2024-12-04T20:28:50,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742132_1308 (size=9857) 2024-12-04T20:28:50,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/82f1fe18be6c4ef2b229c91ca7853b8f 2024-12-04T20:28:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742133_1309 (size=12949) 2024-12-04T20:28:50,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/8fa17675b23640a8ab1e0e8c69acc370 is 50, key is test_row_0/B:col10/1733344130286/Put/seqid=0 2024-12-04T20:28:50,858 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/ccfb902d9fdd4d0c94188575c37f6088 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/ccfb902d9fdd4d0c94188575c37f6088 2024-12-04T20:28:50,863 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into ccfb902d9fdd4d0c94188575c37f6088(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:50,863 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:50,863 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344130759; duration=0sec 2024-12-04T20:28:50,863 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:50,864 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:50,864 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:50,865 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:50,865 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:50,865 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,865 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6492a7d2ffd8418b850fb9bcd3a56927, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/0b64794ea87d4472a46027d52fb371fc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/89d78c39d5dc4c04b7d1c3ecafae74c7] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.6 K 2024-12-04T20:28:50,866 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6492a7d2ffd8418b850fb9bcd3a56927, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733344128339 2024-12-04T20:28:50,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742134_1310 (size=12949) 2024-12-04T20:28:50,866 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b64794ea87d4472a46027d52fb371fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733344128983 2024-12-04T20:28:50,867 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d78c39d5dc4c04b7d1c3ecafae74c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733344129634 2024-12-04T20:28:50,875 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/c4619946d254446a8a133148795f4155 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/c4619946d254446a8a133148795f4155 2024-12-04T20:28:50,881 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into c4619946d254446a8a133148795f4155(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:50,881 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:50,881 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344130759; duration=0sec 2024-12-04T20:28:50,881 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:50,881 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:50,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742135_1311 (size=9857) 2024-12-04T20:28:50,883 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/8fa17675b23640a8ab1e0e8c69acc370 2024-12-04T20:28:50,895 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:50,895 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/62300422a2ea478786068baab2ff4bd9 is 50, key is test_row_0/C:col10/1733344129636/Put/seqid=0 2024-12-04T20:28:50,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/f00db18568d44785af0aedd569c6d53c is 50, key is test_row_0/C:col10/1733344130286/Put/seqid=0 2024-12-04T20:28:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:50,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T20:28:50,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742136_1312 (size=12949) 2024-12-04T20:28:50,929 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/62300422a2ea478786068baab2ff4bd9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/62300422a2ea478786068baab2ff4bd9 2024-12-04T20:28:50,937 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 62300422a2ea478786068baab2ff4bd9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:50,937 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:50,937 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344130759; duration=0sec 2024-12-04T20:28:50,937 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:50,937 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344190933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344190934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344190938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344190942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344190946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:50,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742137_1313 (size=9857) 2024-12-04T20:28:50,948 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/f00db18568d44785af0aedd569c6d53c 2024-12-04T20:28:50,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/82f1fe18be6c4ef2b229c91ca7853b8f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/82f1fe18be6c4ef2b229c91ca7853b8f 2024-12-04T20:28:50,961 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/82f1fe18be6c4ef2b229c91ca7853b8f, entries=100, sequenceid=510, filesize=9.6 K 2024-12-04T20:28:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/8fa17675b23640a8ab1e0e8c69acc370 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8fa17675b23640a8ab1e0e8c69acc370 2024-12-04T20:28:50,968 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8fa17675b23640a8ab1e0e8c69acc370, entries=100, sequenceid=510, filesize=9.6 K 2024-12-04T20:28:50,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/f00db18568d44785af0aedd569c6d53c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/f00db18568d44785af0aedd569c6d53c 2024-12-04T20:28:50,974 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/f00db18568d44785af0aedd569c6d53c, entries=100, sequenceid=510, filesize=9.6 K 2024-12-04T20:28:50,975 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 2687aa8f5064ce602682f5c7138994be in 214ms, sequenceid=510, compaction requested=false 2024-12-04T20:28:50,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:50,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:50,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-04T20:28:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-04T20:28:50,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-04T20:28:50,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 680 msec 2024-12-04T20:28:50,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 687 msec 2024-12-04T20:28:51,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-04T20:28:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:51,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5eeea1f4d8864bb1a9039a525ae9467c is 50, key is test_row_0/A:col10/1733344130938/Put/seqid=0 2024-12-04T20:28:51,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344191062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344191063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344191064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344191065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344191065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742138_1314 (size=17181) 2024-12-04T20:28:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344191171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344191172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344191172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344191182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344191182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344191382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344191382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344191382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344191391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344191392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T20:28:51,431 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:51,432 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-04T20:28:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T20:28:51,433 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:51,434 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:51,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:51,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5eeea1f4d8864bb1a9039a525ae9467c 2024-12-04T20:28:51,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/361fc676496c40999546dfc8bebfd1d6 is 50, key is test_row_0/B:col10/1733344130938/Put/seqid=0 2024-12-04T20:28:51,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742139_1315 (size=12301) 2024-12-04T20:28:51,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/361fc676496c40999546dfc8bebfd1d6 2024-12-04T20:28:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T20:28:51,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/dccb61b5a5f84537b6f4014ebc1a1ada is 50, key is test_row_0/C:col10/1733344130938/Put/seqid=0 2024-12-04T20:28:51,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742140_1316 (size=12301) 2024-12-04T20:28:51,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T20:28:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:51,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344191692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344191692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344191692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344191701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344191701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:51,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T20:28:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T20:28:51,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T20:28:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:51,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/dccb61b5a5f84537b6f4014ebc1a1ada 2024-12-04T20:28:51,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5eeea1f4d8864bb1a9039a525ae9467c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5eeea1f4d8864bb1a9039a525ae9467c 2024-12-04T20:28:51,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5eeea1f4d8864bb1a9039a525ae9467c, entries=250, sequenceid=535, filesize=16.8 K 2024-12-04T20:28:51,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/361fc676496c40999546dfc8bebfd1d6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/361fc676496c40999546dfc8bebfd1d6 2024-12-04T20:28:51,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/361fc676496c40999546dfc8bebfd1d6, entries=150, sequenceid=535, filesize=12.0 K 2024-12-04T20:28:51,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/dccb61b5a5f84537b6f4014ebc1a1ada as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dccb61b5a5f84537b6f4014ebc1a1ada 2024-12-04T20:28:51,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dccb61b5a5f84537b6f4014ebc1a1ada, entries=150, sequenceid=535, filesize=12.0 K 2024-12-04T20:28:51,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 2687aa8f5064ce602682f5c7138994be in 938ms, sequenceid=535, compaction requested=true 2024-12-04T20:28:51,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:51,991 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:51,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:51,992 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39987 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:51,992 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:51,992 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,993 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/c4619946d254446a8a133148795f4155, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/82f1fe18be6c4ef2b229c91ca7853b8f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5eeea1f4d8864bb1a9039a525ae9467c] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=39.0 K 2024-12-04T20:28:51,993 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4619946d254446a8a133148795f4155, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733344129634 2024-12-04T20:28:51,993 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f1fe18be6c4ef2b229c91ca7853b8f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733344130283 2024-12-04T20:28:51,994 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eeea1f4d8864bb1a9039a525ae9467c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733344130929 2024-12-04T20:28:51,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:51,994 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:51,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:51,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:51,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:51,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:51,996 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:51,996 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:51,996 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:51,996 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/ccfb902d9fdd4d0c94188575c37f6088, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8fa17675b23640a8ab1e0e8c69acc370, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/361fc676496c40999546dfc8bebfd1d6] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=34.3 K 2024-12-04T20:28:51,997 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting ccfb902d9fdd4d0c94188575c37f6088, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733344129634 2024-12-04T20:28:51,997 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa17675b23640a8ab1e0e8c69acc370, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733344130283 2024-12-04T20:28:51,998 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 361fc676496c40999546dfc8bebfd1d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733344130929 2024-12-04T20:28:52,004 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:52,005 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6cf5a4673689413cb4b1d278d5cce271 is 50, key is test_row_0/A:col10/1733344130938/Put/seqid=0 2024-12-04T20:28:52,009 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:52,009 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/96a5c9c1f91f41d7a941afdea000e79b is 50, key is test_row_0/B:col10/1733344130938/Put/seqid=0 2024-12-04T20:28:52,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742141_1317 (size=13051) 2024-12-04T20:28:52,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T20:28:52,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:52,054 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T20:28:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:52,056 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6cf5a4673689413cb4b1d278d5cce271 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6cf5a4673689413cb4b1d278d5cce271 2024-12-04T20:28:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T20:28:52,062 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into 6cf5a4673689413cb4b1d278d5cce271(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:52,062 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:52,063 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344131991; duration=0sec 2024-12-04T20:28:52,063 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:52,063 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:52,063 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742142_1318 (size=13051) 2024-12-04T20:28:52,066 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:52,069 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:52,070 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:52,070 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/96a5c9c1f91f41d7a941afdea000e79b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/96a5c9c1f91f41d7a941afdea000e79b 2024-12-04T20:28:52,071 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/62300422a2ea478786068baab2ff4bd9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/f00db18568d44785af0aedd569c6d53c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dccb61b5a5f84537b6f4014ebc1a1ada] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=34.3 K 2024-12-04T20:28:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5a0c7124511a4dc79fbe4f8ee6a23bdb is 50, key is test_row_0/A:col10/1733344131062/Put/seqid=0 2024-12-04T20:28:52,075 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62300422a2ea478786068baab2ff4bd9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733344129634 2024-12-04T20:28:52,076 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f00db18568d44785af0aedd569c6d53c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733344130283 2024-12-04T20:28:52,076 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into 96a5c9c1f91f41d7a941afdea000e79b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:52,076 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:52,076 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344131994; duration=0sec 2024-12-04T20:28:52,076 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:52,076 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:52,076 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting dccb61b5a5f84537b6f4014ebc1a1ada, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733344130929 2024-12-04T20:28:52,085 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:52,085 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/1ca7b1eeed114395ae8dfd6ff1866c83 is 50, key is test_row_0/C:col10/1733344130938/Put/seqid=0 2024-12-04T20:28:52,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742143_1319 (size=12301) 2024-12-04T20:28:52,111 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5a0c7124511a4dc79fbe4f8ee6a23bdb 2024-12-04T20:28:52,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742144_1320 (size=13051) 2024-12-04T20:28:52,124 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/1ca7b1eeed114395ae8dfd6ff1866c83 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1ca7b1eeed114395ae8dfd6ff1866c83 2024-12-04T20:28:52,131 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 1ca7b1eeed114395ae8dfd6ff1866c83(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:52,131 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:52,131 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344131995; duration=0sec 2024-12-04T20:28:52,131 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:52,131 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:52,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/e775cc7e20a442bf9a14b6086510f5ee is 50, key is test_row_0/B:col10/1733344131062/Put/seqid=0 2024-12-04T20:28:52,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742145_1321 (size=12301) 2024-12-04T20:28:52,156 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/e775cc7e20a442bf9a14b6086510f5ee 2024-12-04T20:28:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/dceaedcd0fd94b29af3b4a981b16bffb is 50, key is test_row_0/C:col10/1733344131062/Put/seqid=0 2024-12-04T20:28:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:52,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:52,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742146_1322 (size=12301) 2024-12-04T20:28:52,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/dceaedcd0fd94b29af3b4a981b16bffb 2024-12-04T20:28:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5a0c7124511a4dc79fbe4f8ee6a23bdb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5a0c7124511a4dc79fbe4f8ee6a23bdb 2024-12-04T20:28:52,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5a0c7124511a4dc79fbe4f8ee6a23bdb, entries=150, sequenceid=546, filesize=12.0 K 2024-12-04T20:28:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/e775cc7e20a442bf9a14b6086510f5ee as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e775cc7e20a442bf9a14b6086510f5ee 2024-12-04T20:28:52,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344132235, max timestamp=1733344132240Num uniques 859; 2024-12-04T20:28:52,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344132235, max timestamp=1733344132240Num uniques 859; 2024-12-04T20:28:52,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:52,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:52,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344132235, max timestamp=1733344132240Num uniques 859; 2024-12-04T20:28:52,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:28:52,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e775cc7e20a442bf9a14b6086510f5ee, entries=150, sequenceid=546, filesize=12.0 K 2024-12-04T20:28:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/dceaedcd0fd94b29af3b4a981b16bffb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dceaedcd0fd94b29af3b4a981b16bffb 2024-12-04T20:28:52,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dceaedcd0fd94b29af3b4a981b16bffb, entries=150, sequenceid=546, filesize=12.0 K 2024-12-04T20:28:52,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=80.51 KB/82440 for 2687aa8f5064ce602682f5c7138994be in 200ms, sequenceid=546, compaction requested=false 2024-12-04T20:28:52,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:52,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:52,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-04T20:28:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-04T20:28:52,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=87.22 KB heapSize=245.53 KB 2024-12-04T20:28:52,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:52,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:52,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:52,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:52,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:52,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:52,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-04T20:28:52,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 820 msec 2024-12-04T20:28:52,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 825 msec 2024-12-04T20:28:52,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d2f977eacb8744ce9edc6f7f15721df6 is 50, key is test_row_0/A:col10/1733344132247/Put/seqid=0 2024-12-04T20:28:52,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742147_1323 (size=17181) 2024-12-04T20:28:52,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d2f977eacb8744ce9edc6f7f15721df6 2024-12-04T20:28:52,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344192273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344192273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344192278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/469fa75eb4714bb3a99ecac8b9b4df92 is 50, key is test_row_0/B:col10/1733344132247/Put/seqid=0 2024-12-04T20:28:52,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344192279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344192280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742148_1324 (size=12301) 2024-12-04T20:28:52,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344192382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344192391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344192391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344192391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344192391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T20:28:52,572 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:52,572 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-04T20:28:52,574 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T20:28:52,574 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:52,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:52,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344192592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344192601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344192601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344192601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344192601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T20:28:52,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/469fa75eb4714bb3a99ecac8b9b4df92 2024-12-04T20:28:52,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/a7812ecbce6544e98199b40c43b7fd8f is 50, key is test_row_0/C:col10/1733344132247/Put/seqid=0 2024-12-04T20:28:52,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742149_1325 (size=12301) 2024-12-04T20:28:52,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T20:28:52,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:52,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:52,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:52,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:52,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:52,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T20:28:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:52,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T20:28:52,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344192902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344192912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344192912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344192912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344192912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T20:28:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:53,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:53,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:53,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/a7812ecbce6544e98199b40c43b7fd8f 2024-12-04T20:28:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/d2f977eacb8744ce9edc6f7f15721df6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d2f977eacb8744ce9edc6f7f15721df6 2024-12-04T20:28:53,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d2f977eacb8744ce9edc6f7f15721df6, entries=250, sequenceid=580, filesize=16.8 K 2024-12-04T20:28:53,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/469fa75eb4714bb3a99ecac8b9b4df92 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/469fa75eb4714bb3a99ecac8b9b4df92 2024-12-04T20:28:53,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/469fa75eb4714bb3a99ecac8b9b4df92, entries=150, sequenceid=580, filesize=12.0 K 2024-12-04T20:28:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/a7812ecbce6544e98199b40c43b7fd8f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/a7812ecbce6544e98199b40c43b7fd8f 2024-12-04T20:28:53,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/a7812ecbce6544e98199b40c43b7fd8f, entries=150, sequenceid=580, filesize=12.0 K 2024-12-04T20:28:53,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~265.41 KB/271776, currentSize=93.93 KB/96180 for 2687aa8f5064ce602682f5c7138994be in 881ms, sequenceid=580, compaction requested=true 2024-12-04T20:28:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:53,135 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:53,135 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:53,136 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:53,136 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:53,136 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:53,137 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6cf5a4673689413cb4b1d278d5cce271, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5a0c7124511a4dc79fbe4f8ee6a23bdb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d2f977eacb8744ce9edc6f7f15721df6] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=41.5 K 2024-12-04T20:28:53,137 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:53,137 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf5a4673689413cb4b1d278d5cce271, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733344130929 2024-12-04T20:28:53,137 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:53,137 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:53,137 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/96a5c9c1f91f41d7a941afdea000e79b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e775cc7e20a442bf9a14b6086510f5ee, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/469fa75eb4714bb3a99ecac8b9b4df92] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.8 K 2024-12-04T20:28:53,138 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a5c9c1f91f41d7a941afdea000e79b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733344130929 2024-12-04T20:28:53,138 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0c7124511a4dc79fbe4f8ee6a23bdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1733344131059 2024-12-04T20:28:53,138 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e775cc7e20a442bf9a14b6086510f5ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1733344131059 2024-12-04T20:28:53,138 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2f977eacb8744ce9edc6f7f15721df6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733344132247 2024-12-04T20:28:53,139 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 469fa75eb4714bb3a99ecac8b9b4df92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733344132247 2024-12-04T20:28:53,147 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:53,148 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/ab7585f70f8d436ebaa781ba21af5c95 is 50, key is test_row_0/A:col10/1733344132247/Put/seqid=0 2024-12-04T20:28:53,151 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:53,152 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/0296b32b38c9460585ee0ec359c7645c is 50, key is test_row_0/B:col10/1733344132247/Put/seqid=0 2024-12-04T20:28:53,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742150_1326 (size=13153) 2024-12-04T20:28:53,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T20:28:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:53,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T20:28:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:53,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742151_1327 (size=13153) 2024-12-04T20:28:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T20:28:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/1a253724215f4ebe80a3eb65412a2a6f is 50, key is test_row_0/A:col10/1733344132278/Put/seqid=0 2024-12-04T20:28:53,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742152_1328 (size=12301) 2024-12-04T20:28:53,211 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/1a253724215f4ebe80a3eb65412a2a6f 2024-12-04T20:28:53,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/7149f16ccf544425913a9eb297803440 is 50, key is test_row_0/B:col10/1733344132278/Put/seqid=0 2024-12-04T20:28:53,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742153_1329 (size=12301) 2024-12-04T20:28:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:53,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344193430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344193430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344193430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344193431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344193433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344193541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344193542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344193542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344193542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344193542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,589 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/ab7585f70f8d436ebaa781ba21af5c95 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ab7585f70f8d436ebaa781ba21af5c95 2024-12-04T20:28:53,594 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into ab7585f70f8d436ebaa781ba21af5c95(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:53,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:53,595 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344133135; duration=0sec 2024-12-04T20:28:53,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:53,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:53,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:53,596 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:53,596 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:53,596 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:53,596 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1ca7b1eeed114395ae8dfd6ff1866c83, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dceaedcd0fd94b29af3b4a981b16bffb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/a7812ecbce6544e98199b40c43b7fd8f] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.8 K 2024-12-04T20:28:53,596 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca7b1eeed114395ae8dfd6ff1866c83, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733344130929 2024-12-04T20:28:53,597 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting dceaedcd0fd94b29af3b4a981b16bffb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1733344131059 2024-12-04T20:28:53,597 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7812ecbce6544e98199b40c43b7fd8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733344132247 2024-12-04T20:28:53,603 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/0296b32b38c9460585ee0ec359c7645c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0296b32b38c9460585ee0ec359c7645c 2024-12-04T20:28:53,604 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:53,604 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/c32501ae346443568415ca6dc41ba1ec is 50, key is test_row_0/C:col10/1733344132247/Put/seqid=0 2024-12-04T20:28:53,609 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into 0296b32b38c9460585ee0ec359c7645c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:53,609 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:53,609 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344133135; duration=0sec 2024-12-04T20:28:53,609 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:53,609 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:53,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742154_1330 (size=13153) 2024-12-04T20:28:53,631 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/7149f16ccf544425913a9eb297803440 2024-12-04T20:28:53,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/be435898ae4f474490b6f90d4f9190ae is 50, key is test_row_0/C:col10/1733344132278/Put/seqid=0 2024-12-04T20:28:53,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742155_1331 (size=12301) 2024-12-04T20:28:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T20:28:53,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344193752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344193752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344193752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344193754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:53,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344193755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:53,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:53,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,027 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/c32501ae346443568415ca6dc41ba1ec as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c32501ae346443568415ca6dc41ba1ec 2024-12-04T20:28:54,032 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into c32501ae346443568415ca6dc41ba1ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:54,032 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:54,032 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344133135; duration=0sec 2024-12-04T20:28:54,033 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:54,033 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:54,042 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/be435898ae4f474490b6f90d4f9190ae 2024-12-04T20:28:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/1a253724215f4ebe80a3eb65412a2a6f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/1a253724215f4ebe80a3eb65412a2a6f 2024-12-04T20:28:54,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/1a253724215f4ebe80a3eb65412a2a6f, entries=150, sequenceid=597, filesize=12.0 K 2024-12-04T20:28:54,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/7149f16ccf544425913a9eb297803440 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/7149f16ccf544425913a9eb297803440 2024-12-04T20:28:54,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/7149f16ccf544425913a9eb297803440, entries=150, sequenceid=597, filesize=12.0 K 2024-12-04T20:28:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/be435898ae4f474490b6f90d4f9190ae as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/be435898ae4f474490b6f90d4f9190ae 2024-12-04T20:28:54,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344194062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344194062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344194062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344194062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344194062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,069 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/be435898ae4f474490b6f90d4f9190ae, entries=150, sequenceid=597, filesize=12.0 K 2024-12-04T20:28:54,071 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2687aa8f5064ce602682f5c7138994be in 886ms, sequenceid=597, compaction requested=false 2024-12-04T20:28:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-04T20:28:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-04T20:28:54,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-04T20:28:54,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4980 sec 2024-12-04T20:28:54,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5030 sec 2024-12-04T20:28:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:54,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T20:28:54,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:54,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:54,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:54,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:54,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:54,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:54,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/de131a76563740f5bfa5b537bae7a5ae is 50, key is test_row_0/A:col10/1733344134572/Put/seqid=0 2024-12-04T20:28:54,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742156_1332 (size=12301) 2024-12-04T20:28:54,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344194583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344194587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344194587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344194587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344194588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344194691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344194702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344194702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344194702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344194702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T20:28:54,721 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:54,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-04T20:28:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T20:28:54,723 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:54,724 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:54,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T20:28:54,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:54,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:54,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344194902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344194912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344194912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344194912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344194912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:54,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/de131a76563740f5bfa5b537bae7a5ae 2024-12-04T20:28:54,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/f8b616345b044349896172fdff601124 is 50, key is test_row_0/B:col10/1733344134572/Put/seqid=0 2024-12-04T20:28:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742157_1333 (size=12301) 2024-12-04T20:28:55,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T20:28:55,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344195212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344195221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344195221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344195221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344195222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T20:28:55,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/f8b616345b044349896172fdff601124 2024-12-04T20:28:55,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/13e14df27f13484db6d2adecc25daa7a is 50, key is test_row_0/C:col10/1733344134572/Put/seqid=0 2024-12-04T20:28:55,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742158_1334 (size=12301) 2024-12-04T20:28:55,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:55,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:55,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344195721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344195732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344195732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344195732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344195732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:55,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:55,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/13e14df27f13484db6d2adecc25daa7a 2024-12-04T20:28:55,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/de131a76563740f5bfa5b537bae7a5ae as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/de131a76563740f5bfa5b537bae7a5ae 2024-12-04T20:28:55,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/de131a76563740f5bfa5b537bae7a5ae, entries=150, sequenceid=618, filesize=12.0 K 2024-12-04T20:28:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T20:28:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/f8b616345b044349896172fdff601124 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f8b616345b044349896172fdff601124 2024-12-04T20:28:55,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f8b616345b044349896172fdff601124, entries=150, sequenceid=618, filesize=12.0 K 2024-12-04T20:28:55,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/13e14df27f13484db6d2adecc25daa7a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/13e14df27f13484db6d2adecc25daa7a 2024-12-04T20:28:55,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/13e14df27f13484db6d2adecc25daa7a, entries=150, sequenceid=618, filesize=12.0 K 2024-12-04T20:28:55,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 2687aa8f5064ce602682f5c7138994be in 1315ms, sequenceid=618, compaction requested=true 2024-12-04T20:28:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:55,888 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:55,888 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:55,890 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:55,890 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:55,890 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,890 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0296b32b38c9460585ee0ec359c7645c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/7149f16ccf544425913a9eb297803440, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f8b616345b044349896172fdff601124] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.9 K 2024-12-04T20:28:55,891 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:55,891 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:55,891 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,891 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ab7585f70f8d436ebaa781ba21af5c95, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/1a253724215f4ebe80a3eb65412a2a6f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/de131a76563740f5bfa5b537bae7a5ae] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.9 K 2024-12-04T20:28:55,891 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0296b32b38c9460585ee0ec359c7645c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733344132247 2024-12-04T20:28:55,891 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab7585f70f8d436ebaa781ba21af5c95, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733344132247 2024-12-04T20:28:55,892 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7149f16ccf544425913a9eb297803440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733344132271 2024-12-04T20:28:55,892 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a253724215f4ebe80a3eb65412a2a6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733344132271 2024-12-04T20:28:55,892 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b616345b044349896172fdff601124, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1733344133431 2024-12-04T20:28:55,892 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting de131a76563740f5bfa5b537bae7a5ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1733344133431 2024-12-04T20:28:55,923 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:55,924 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9bf38c4d949b4c38ac33ba912903e827 is 50, key is test_row_0/B:col10/1733344134572/Put/seqid=0 2024-12-04T20:28:55,935 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:55,936 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/33c3c34e2fb1449abff2dd49e84f51c1 is 50, key is test_row_0/A:col10/1733344134572/Put/seqid=0 2024-12-04T20:28:55,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T20:28:55,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,951 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T20:28:55,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:55,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:55,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:55,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:55,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:55,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:55,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742159_1335 (size=13255) 2024-12-04T20:28:55,961 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/9bf38c4d949b4c38ac33ba912903e827 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9bf38c4d949b4c38ac33ba912903e827 2024-12-04T20:28:55,969 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into 9bf38c4d949b4c38ac33ba912903e827(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:55,969 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:55,969 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344135888; duration=0sec 2024-12-04T20:28:55,970 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:55,970 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:55,970 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:55,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5b6bdc9254a6487797b5b765d588c16e is 50, key is test_row_0/A:col10/1733344134579/Put/seqid=0 2024-12-04T20:28:55,985 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:55,986 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:55,986 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:55,986 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c32501ae346443568415ca6dc41ba1ec, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/be435898ae4f474490b6f90d4f9190ae, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/13e14df27f13484db6d2adecc25daa7a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=36.9 K 2024-12-04T20:28:55,986 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c32501ae346443568415ca6dc41ba1ec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733344132247 2024-12-04T20:28:55,986 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting be435898ae4f474490b6f90d4f9190ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733344132271 2024-12-04T20:28:55,987 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 13e14df27f13484db6d2adecc25daa7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1733344133431 2024-12-04T20:28:55,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742160_1336 (size=13255) 2024-12-04T20:28:56,025 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:56,025 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/4ebde424c8f64b7eb7b0766e1c7dcee7 is 50, key is test_row_0/C:col10/1733344134572/Put/seqid=0 2024-12-04T20:28:56,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742161_1337 (size=12301) 2024-12-04T20:28:56,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742162_1338 (size=13255) 2024-12-04T20:28:56,056 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/4ebde424c8f64b7eb7b0766e1c7dcee7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4ebde424c8f64b7eb7b0766e1c7dcee7 2024-12-04T20:28:56,062 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into 4ebde424c8f64b7eb7b0766e1c7dcee7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:56,062 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:56,062 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344135888; duration=0sec 2024-12-04T20:28:56,062 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:56,062 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:56,413 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/33c3c34e2fb1449abff2dd49e84f51c1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/33c3c34e2fb1449abff2dd49e84f51c1 2024-12-04T20:28:56,418 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into 33c3c34e2fb1449abff2dd49e84f51c1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:56,418 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:56,418 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344135888; duration=0sec 2024-12-04T20:28:56,419 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:56,419 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:56,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5b6bdc9254a6487797b5b765d588c16e 2024-12-04T20:28:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/617f33f9e11047cda526cc1a56ed14f0 is 50, key is test_row_0/B:col10/1733344134579/Put/seqid=0 2024-12-04T20:28:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742163_1339 (size=12301) 2024-12-04T20:28:56,508 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/617f33f9e11047cda526cc1a56ed14f0 2024-12-04T20:28:56,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/4d1060a36f98497887990f21f92039ed is 50, key is test_row_0/C:col10/1733344134579/Put/seqid=0 2024-12-04T20:28:56,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742164_1340 (size=12301) 2024-12-04T20:28:56,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/4d1060a36f98497887990f21f92039ed 2024-12-04T20:28:56,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/5b6bdc9254a6487797b5b765d588c16e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5b6bdc9254a6487797b5b765d588c16e 2024-12-04T20:28:56,597 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5b6bdc9254a6487797b5b765d588c16e, entries=150, sequenceid=633, filesize=12.0 K 2024-12-04T20:28:56,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/617f33f9e11047cda526cc1a56ed14f0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/617f33f9e11047cda526cc1a56ed14f0 2024-12-04T20:28:56,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/617f33f9e11047cda526cc1a56ed14f0, entries=150, sequenceid=633, filesize=12.0 K 2024-12-04T20:28:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/4d1060a36f98497887990f21f92039ed as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4d1060a36f98497887990f21f92039ed 2024-12-04T20:28:56,622 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4d1060a36f98497887990f21f92039ed, entries=150, sequenceid=633, filesize=12.0 K 2024-12-04T20:28:56,623 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 2687aa8f5064ce602682f5c7138994be in 671ms, sequenceid=633, compaction requested=false 2024-12-04T20:28:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-04T20:28:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-04T20:28:56,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-04T20:28:56,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9000 sec 2024-12-04T20:28:56,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.9070 sec 2024-12-04T20:28:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:56,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:28:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:56,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6aa5fcb03f88445ba2f6dc25b82b9ccd is 50, key is test_row_0/A:col10/1733344136747/Put/seqid=0 2024-12-04T20:28:56,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742165_1341 (size=12301) 2024-12-04T20:28:56,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344136774, max timestamp=1733344136785Num uniques 859; 2024-12-04T20:28:56,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344136774, max timestamp=1733344136785Num uniques 859; 2024-12-04T20:28:56,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:28:56,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:28:56,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733344136774, max timestamp=1733344136785Num uniques 859; 2024-12-04T20:28:56,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:28:56,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6aa5fcb03f88445ba2f6dc25b82b9ccd 2024-12-04T20:28:56,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/217d0e6679b146debc30304aa9cc739b is 50, key is test_row_0/B:col10/1733344136747/Put/seqid=0 2024-12-04T20:28:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344196816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344196821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344196823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344196825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344196825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742166_1342 (size=12301) 2024-12-04T20:28:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T20:28:56,872 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:28:56,872 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:28:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-04T20:28:56,875 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:28:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T20:28:56,875 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:28:56,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:28:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344196931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344196931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344196942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344196942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344196942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T20:28:57,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T20:28:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344197141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344197141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344197151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344197152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344197152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T20:28:57,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:57,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T20:28:57,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/217d0e6679b146debc30304aa9cc739b 2024-12-04T20:28:57,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/c05b1e26d4ab4be6a0fb81497d9b7d1f is 50, key is test_row_0/C:col10/1733344136747/Put/seqid=0 2024-12-04T20:28:57,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742167_1343 (size=12301) 2024-12-04T20:28:57,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T20:28:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344197452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344197452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344197462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344197462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344197462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T20:28:57,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:57,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T20:28:57,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T20:28:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:28:57,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/c05b1e26d4ab4be6a0fb81497d9b7d1f 2024-12-04T20:28:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6aa5fcb03f88445ba2f6dc25b82b9ccd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6aa5fcb03f88445ba2f6dc25b82b9ccd 2024-12-04T20:28:57,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6aa5fcb03f88445ba2f6dc25b82b9ccd, entries=150, sequenceid=646, filesize=12.0 K 2024-12-04T20:28:57,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/217d0e6679b146debc30304aa9cc739b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/217d0e6679b146debc30304aa9cc739b 2024-12-04T20:28:57,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/217d0e6679b146debc30304aa9cc739b, entries=150, sequenceid=646, filesize=12.0 K 2024-12-04T20:28:57,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/c05b1e26d4ab4be6a0fb81497d9b7d1f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c05b1e26d4ab4be6a0fb81497d9b7d1f 2024-12-04T20:28:57,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c05b1e26d4ab4be6a0fb81497d9b7d1f, entries=150, sequenceid=646, filesize=12.0 K 2024-12-04T20:28:57,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2687aa8f5064ce602682f5c7138994be in 953ms, sequenceid=646, compaction requested=true 2024-12-04T20:28:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:28:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:57,701 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:57,702 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:57,702 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:57,703 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:28:57,703 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:57,703 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:28:57,703 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:28:57,703 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:28:57,703 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/33c3c34e2fb1449abff2dd49e84f51c1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5b6bdc9254a6487797b5b765d588c16e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6aa5fcb03f88445ba2f6dc25b82b9ccd] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=37.0 K 2024-12-04T20:28:57,703 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9bf38c4d949b4c38ac33ba912903e827, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/617f33f9e11047cda526cc1a56ed14f0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/217d0e6679b146debc30304aa9cc739b] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=37.0 K 2024-12-04T20:28:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:57,703 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c3c34e2fb1449abff2dd49e84f51c1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1733344133431 2024-12-04T20:28:57,703 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf38c4d949b4c38ac33ba912903e827, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1733344133431 2024-12-04T20:28:57,703 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6bdc9254a6487797b5b765d588c16e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=633, earliestPutTs=1733344134579 2024-12-04T20:28:57,704 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 617f33f9e11047cda526cc1a56ed14f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=633, earliestPutTs=1733344134579 2024-12-04T20:28:57,704 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 217d0e6679b146debc30304aa9cc739b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733344136743 2024-12-04T20:28:57,704 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aa5fcb03f88445ba2f6dc25b82b9ccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733344136743 2024-12-04T20:28:57,711 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:57,712 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/8dac819277474d34bb06b4794152cf4e is 50, key is test_row_0/B:col10/1733344136747/Put/seqid=0 2024-12-04T20:28:57,716 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:57,716 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/db6f109b5ebb4ec099978328dcf92fc2 is 50, key is test_row_0/A:col10/1733344136747/Put/seqid=0 2024-12-04T20:28:57,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742168_1344 (size=13357) 2024-12-04T20:28:57,735 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/8dac819277474d34bb06b4794152cf4e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8dac819277474d34bb06b4794152cf4e 2024-12-04T20:28:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742169_1345 (size=13357) 2024-12-04T20:28:57,744 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into 8dac819277474d34bb06b4794152cf4e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:57,744 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:57,744 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344137701; duration=0sec 2024-12-04T20:28:57,745 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:28:57,745 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:28:57,745 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:28:57,746 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:28:57,746 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:28:57,747 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,747 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4ebde424c8f64b7eb7b0766e1c7dcee7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4d1060a36f98497887990f21f92039ed, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c05b1e26d4ab4be6a0fb81497d9b7d1f] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=37.0 K 2024-12-04T20:28:57,747 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ebde424c8f64b7eb7b0766e1c7dcee7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1733344133431 2024-12-04T20:28:57,747 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1060a36f98497887990f21f92039ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=633, earliestPutTs=1733344134579 2024-12-04T20:28:57,748 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/db6f109b5ebb4ec099978328dcf92fc2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/db6f109b5ebb4ec099978328dcf92fc2 2024-12-04T20:28:57,748 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c05b1e26d4ab4be6a0fb81497d9b7d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733344136743 2024-12-04T20:28:57,753 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into db6f109b5ebb4ec099978328dcf92fc2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:57,754 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:57,755 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344137701; duration=0sec 2024-12-04T20:28:57,755 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:57,755 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:28:57,758 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:28:57,759 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/c2006e5428944725a91222fd62541340 is 50, key is test_row_0/C:col10/1733344136747/Put/seqid=0 2024-12-04T20:28:57,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742170_1346 (size=13357) 2024-12-04T20:28:57,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T20:28:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:57,798 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T20:28:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:57,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:28:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/8a733d81441746fa814f844a4a957b22 is 50, key is test_row_0/A:col10/1733344136819/Put/seqid=0 2024-12-04T20:28:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742171_1347 (size=12301) 2024-12-04T20:28:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:57,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. as already flushing 2024-12-04T20:28:57,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344197972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344197973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344197976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344197978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:57,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344197978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T20:28:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344198081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344198081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344198091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344198092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344198092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,195 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/c2006e5428944725a91222fd62541340 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c2006e5428944725a91222fd62541340 2024-12-04T20:28:58,200 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into c2006e5428944725a91222fd62541340(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:28:58,200 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:58,200 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344137703; duration=0sec 2024-12-04T20:28:58,200 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:28:58,200 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:28:58,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/8a733d81441746fa814f844a4a957b22 2024-12-04T20:28:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/e8a410bdd244457abebcd8cff825f3ad is 50, key is test_row_0/B:col10/1733344136819/Put/seqid=0 2024-12-04T20:28:58,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742172_1348 (size=12301) 2024-12-04T20:28:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344198292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344198292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344198301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344198301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344198301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344198602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344198602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344198611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344198612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344198612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:58,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:58,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:58,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/e8a410bdd244457abebcd8cff825f3ad 2024-12-04T20:28:58,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/74267d05c7a54e77a75c83f32ff3caf1 is 50, key is test_row_0/C:col10/1733344136819/Put/seqid=0 2024-12-04T20:28:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742173_1349 (size=12301) 2024-12-04T20:28:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T20:28:59,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/74267d05c7a54e77a75c83f32ff3caf1 2024-12-04T20:28:59,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/8a733d81441746fa814f844a4a957b22 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8a733d81441746fa814f844a4a957b22 2024-12-04T20:28:59,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8a733d81441746fa814f844a4a957b22, entries=150, sequenceid=684, filesize=12.0 K 2024-12-04T20:28:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/e8a410bdd244457abebcd8cff825f3ad as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e8a410bdd244457abebcd8cff825f3ad 2024-12-04T20:28:59,072 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e8a410bdd244457abebcd8cff825f3ad, entries=150, sequenceid=684, filesize=12.0 K 2024-12-04T20:28:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/74267d05c7a54e77a75c83f32ff3caf1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/74267d05c7a54e77a75c83f32ff3caf1 2024-12-04T20:28:59,076 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/74267d05c7a54e77a75c83f32ff3caf1, entries=150, sequenceid=684, filesize=12.0 K 2024-12-04T20:28:59,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 2687aa8f5064ce602682f5c7138994be in 1278ms, sequenceid=684, compaction requested=false 2024-12-04T20:28:59,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:28:59,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:28:59,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-04T20:28:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-04T20:28:59,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-04T20:28:59,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2020 sec 2024-12-04T20:28:59,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2090 sec 2024-12-04T20:28:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:28:59,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T20:28:59,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:28:59,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:59,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:28:59,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:59,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:28:59,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:28:59,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14 is 50, key is test_row_0/A:col10/1733344137974/Put/seqid=0 2024-12-04T20:28:59,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742174_1350 (size=14741) 2024-12-04T20:28:59,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344199137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344199137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344199140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344199141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344199142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344199252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344199252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344199253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344199253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344199257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344199461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344199461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344199461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344199461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344199472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=697 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14 2024-12-04T20:28:59,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/739df2c7363b455090aa6637582ac665 is 50, key is test_row_0/B:col10/1733344137974/Put/seqid=0 2024-12-04T20:28:59,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742175_1351 (size=12301) 2024-12-04T20:28:59,701 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-12-04T20:28:59,701 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:59,701 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,701 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:59,702 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-12-04T20:28:59,702 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:59,702 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,702 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:59,703 INFO [Thread-1176 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1176. 2024-12-04T20:28:59,703 DEBUG [Thread-1176 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:59,703 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,703 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:59,704 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-12-04T20:28:59,704 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:59,704 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,704 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:59,706 INFO [Thread-1178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1178. 2024-12-04T20:28:59,706 DEBUG [Thread-1178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:28:59,706 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:28:59,707 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:28:59,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344199771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344199772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344199771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344199772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:28:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344199782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:28:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:28:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:28:59,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=697 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/739df2c7363b455090aa6637582ac665 2024-12-04T20:28:59,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/7ec59836fa96440d9bd2a323df6d827a is 50, key is test_row_0/C:col10/1733344137974/Put/seqid=0 2024-12-04T20:28:59,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742176_1352 (size=12301) 2024-12-04T20:29:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42292 deadline: 1733344200282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42276 deadline: 1733344200281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1733344200281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42248 deadline: 1733344200282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:00,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42312 deadline: 1733344200292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:00,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=697 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/7ec59836fa96440d9bd2a323df6d827a 2024-12-04T20:29:00,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14 2024-12-04T20:29:00,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14, entries=200, sequenceid=697, filesize=14.4 K 2024-12-04T20:29:00,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/739df2c7363b455090aa6637582ac665 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/739df2c7363b455090aa6637582ac665 2024-12-04T20:29:00,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/739df2c7363b455090aa6637582ac665, entries=150, sequenceid=697, filesize=12.0 K 2024-12-04T20:29:00,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/7ec59836fa96440d9bd2a323df6d827a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/7ec59836fa96440d9bd2a323df6d827a 2024-12-04T20:29:00,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/7ec59836fa96440d9bd2a323df6d827a, entries=150, sequenceid=697, filesize=12.0 K 2024-12-04T20:29:00,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2687aa8f5064ce602682f5c7138994be in 1302ms, sequenceid=697, compaction requested=true 2024-12-04T20:29:00,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:29:00,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:00,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:00,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:00,415 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:00,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:00,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2687aa8f5064ce602682f5c7138994be:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:00,415 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:00,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:00,416 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:00,416 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:00,416 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/B is initiating minor compaction (all files) 2024-12-04T20:29:00,416 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/A is initiating minor compaction (all files) 2024-12-04T20:29:00,416 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/B in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:00,416 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/A in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:00,416 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8dac819277474d34bb06b4794152cf4e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e8a410bdd244457abebcd8cff825f3ad, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/739df2c7363b455090aa6637582ac665] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=37.1 K 2024-12-04T20:29:00,416 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/db6f109b5ebb4ec099978328dcf92fc2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8a733d81441746fa814f844a4a957b22, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=39.5 K 2024-12-04T20:29:00,416 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dac819277474d34bb06b4794152cf4e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733344136743 2024-12-04T20:29:00,416 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting db6f109b5ebb4ec099978328dcf92fc2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733344136743 2024-12-04T20:29:00,417 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a733d81441746fa814f844a4a957b22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733344136819 2024-12-04T20:29:00,417 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a410bdd244457abebcd8cff825f3ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733344136819 2024-12-04T20:29:00,417 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37af5aa2ab1b46b0bfcd29bcdbf8ab14, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=697, earliestPutTs=1733344137974 2024-12-04T20:29:00,417 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 739df2c7363b455090aa6637582ac665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=697, earliestPutTs=1733344137974 2024-12-04T20:29:00,422 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:00,422 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/f5d8eb04f2cb41909ac077f8709072ef is 50, key is test_row_0/B:col10/1733344137974/Put/seqid=0 2024-12-04T20:29:00,422 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#A#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:00,423 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/ac98fc1f33ef4a979ed7a02f1dc337b4 is 50, key is test_row_0/A:col10/1733344137974/Put/seqid=0 2024-12-04T20:29:00,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742177_1353 (size=13459) 2024-12-04T20:29:00,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742178_1354 (size=13459) 2024-12-04T20:29:00,829 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/ac98fc1f33ef4a979ed7a02f1dc337b4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ac98fc1f33ef4a979ed7a02f1dc337b4 2024-12-04T20:29:00,829 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/f5d8eb04f2cb41909ac077f8709072ef as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f5d8eb04f2cb41909ac077f8709072ef 2024-12-04T20:29:00,833 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/A of 2687aa8f5064ce602682f5c7138994be into ac98fc1f33ef4a979ed7a02f1dc337b4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:00,833 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/B of 2687aa8f5064ce602682f5c7138994be into f5d8eb04f2cb41909ac077f8709072ef(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:00,833 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:29:00,833 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:29:00,833 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/A, priority=13, startTime=1733344140415; duration=0sec 2024-12-04T20:29:00,833 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/B, priority=13, startTime=1733344140415; duration=0sec 2024-12-04T20:29:00,833 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:00,833 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:A 2024-12-04T20:29:00,833 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:00,833 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:B 2024-12-04T20:29:00,833 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:00,834 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:00,834 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 2687aa8f5064ce602682f5c7138994be/C is initiating minor compaction (all files) 2024-12-04T20:29:00,834 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2687aa8f5064ce602682f5c7138994be/C in TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:00,835 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c2006e5428944725a91222fd62541340, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/74267d05c7a54e77a75c83f32ff3caf1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/7ec59836fa96440d9bd2a323df6d827a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp, totalSize=37.1 K 2024-12-04T20:29:00,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2006e5428944725a91222fd62541340, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733344136743 2024-12-04T20:29:00,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74267d05c7a54e77a75c83f32ff3caf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733344136819 2024-12-04T20:29:00,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec59836fa96440d9bd2a323df6d827a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=697, earliestPutTs=1733344137974 2024-12-04T20:29:00,841 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2687aa8f5064ce602682f5c7138994be#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:00,841 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b1ce3e22b45c44e694ca29797d53eb18 is 50, key is test_row_0/C:col10/1733344137974/Put/seqid=0 2024-12-04T20:29:00,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742179_1355 (size=13459) 2024-12-04T20:29:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T20:29:01,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:01,251 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/b1ce3e22b45c44e694ca29797d53eb18 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b1ce3e22b45c44e694ca29797d53eb18 2024-12-04T20:29:01,256 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2687aa8f5064ce602682f5c7138994be/C of 2687aa8f5064ce602682f5c7138994be into b1ce3e22b45c44e694ca29797d53eb18(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:01,256 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:29:01,256 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., storeName=2687aa8f5064ce602682f5c7138994be/C, priority=13, startTime=1733344140415; duration=0sec 2024-12-04T20:29:01,256 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:01,256 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2687aa8f5064ce602682f5c7138994be:C 2024-12-04T20:29:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:29:01,293 INFO [Thread-1167 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1167. 2024-12-04T20:29:01,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-12-04T20:29:01,293 DEBUG [Thread-1167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:01,293 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,293 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:01,293 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-12-04T20:29:01,293 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:01,293 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,293 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344139117, max timestamp=1733344141291Num uniques 859; 2024-12-04T20:29:01,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344139117, max timestamp=1733344141291Num uniques 859; 2024-12-04T20:29:01,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:01,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:01,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:01,293 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-12-04T20:29:01,294 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:01,294 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,294 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:01,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344139117, max timestamp=1733344141291Num uniques 859; 2024-12-04T20:29:01,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:01,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:29:01,295 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-12-04T20:29:01,295 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:01,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:01,295 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:29:01,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:01,295 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:29:01,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:01,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:01,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6dc5f427998d4e648c1796f09de1db66 is 50, key is test_row_0/A:col10/1733344141291/Put/seqid=0 2024-12-04T20:29:01,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742180_1356 (size=12301) 2024-12-04T20:29:01,303 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-12-04T20:29:01,303 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:01,303 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,303 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 149 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3493 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3466 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3407 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3501 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3490 2024-12-04T20:29:01,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T20:29:01,303 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:29:01,303 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:29:01,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:01,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:01,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T20:29:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T20:29:01,307 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344141307"}]},"ts":"1733344141307"} 2024-12-04T20:29:01,308 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T20:29:01,308 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T20:29:01,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:29:01,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, UNASSIGN}] 2024-12-04T20:29:01,311 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, UNASSIGN 2024-12-04T20:29:01,312 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=2687aa8f5064ce602682f5c7138994be, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:01,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, UNASSIGN because future has completed 2024-12-04T20:29:01,314 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:29:01,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:29:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T20:29:01,466 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:29:01,466 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:29:01,466 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 2687aa8f5064ce602682f5c7138994be, disabling compactions & flushes 2024-12-04T20:29:01,466 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T20:29:01,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6dc5f427998d4e648c1796f09de1db66 2024-12-04T20:29:01,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/5cb374b5670b4518ac627bf75c83c9d5 is 50, key is test_row_0/B:col10/1733344141291/Put/seqid=0 2024-12-04T20:29:01,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742181_1357 (size=12301) 2024-12-04T20:29:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T20:29:02,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/5cb374b5670b4518ac627bf75c83c9d5 2024-12-04T20:29:02,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/266913e961e94dffafe5f9b6b3d57179 is 50, key is test_row_0/C:col10/1733344141291/Put/seqid=0 2024-12-04T20:29:02,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742182_1358 (size=12301) 2024-12-04T20:29:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T20:29:02,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/266913e961e94dffafe5f9b6b3d57179 2024-12-04T20:29:02,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/6dc5f427998d4e648c1796f09de1db66 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6dc5f427998d4e648c1796f09de1db66 2024-12-04T20:29:02,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6dc5f427998d4e648c1796f09de1db66, entries=150, sequenceid=724, filesize=12.0 K 2024-12-04T20:29:02,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/5cb374b5670b4518ac627bf75c83c9d5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5cb374b5670b4518ac627bf75c83c9d5 2024-12-04T20:29:02,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5cb374b5670b4518ac627bf75c83c9d5, entries=150, sequenceid=724, filesize=12.0 K 2024-12-04T20:29:02,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/266913e961e94dffafe5f9b6b3d57179 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/266913e961e94dffafe5f9b6b3d57179 2024-12-04T20:29:02,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/266913e961e94dffafe5f9b6b3d57179, entries=150, sequenceid=724, filesize=12.0 K 2024-12-04T20:29:02,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.95 KB/128976, currentSize=6.71 KB/6870 for 2687aa8f5064ce602682f5c7138994be in 1248ms, sequenceid=724, compaction requested=false 2024-12-04T20:29:02,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2687aa8f5064ce602682f5c7138994be: 2024-12-04T20:29:02,541 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. after waiting 0 ms 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:02,541 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 2687aa8f5064ce602682f5c7138994be 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=A 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=B 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2687aa8f5064ce602682f5c7138994be, store=C 2024-12-04T20:29:02,541 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:02,544 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/01984945867f48c49202d83fa65c2bc0 is 50, key is test_row_0/A:col10/1733344141302/Put/seqid=0 2024-12-04T20:29:02,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742183_1359 (size=7415) 2024-12-04T20:29:02,786 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:29:02,951 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/01984945867f48c49202d83fa65c2bc0 2024-12-04T20:29:02,959 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/1b8587dd1ad549e1aa4b15ff1e16087a is 50, key is test_row_0/B:col10/1733344141302/Put/seqid=0 2024-12-04T20:29:02,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742184_1360 (size=7415) 2024-12-04T20:29:03,366 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/1b8587dd1ad549e1aa4b15ff1e16087a 2024-12-04T20:29:03,380 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/39879f8fdcf24a1f94d637dfda181ab0 is 50, key is test_row_0/C:col10/1733344141302/Put/seqid=0 2024-12-04T20:29:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742185_1361 (size=7415) 2024-12-04T20:29:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T20:29:03,785 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/39879f8fdcf24a1f94d637dfda181ab0 2024-12-04T20:29:03,795 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/A/01984945867f48c49202d83fa65c2bc0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/01984945867f48c49202d83fa65c2bc0 2024-12-04T20:29:03,798 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/01984945867f48c49202d83fa65c2bc0, entries=50, sequenceid=728, filesize=7.2 K 2024-12-04T20:29:03,799 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/B/1b8587dd1ad549e1aa4b15ff1e16087a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1b8587dd1ad549e1aa4b15ff1e16087a 2024-12-04T20:29:03,803 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1b8587dd1ad549e1aa4b15ff1e16087a, entries=50, sequenceid=728, filesize=7.2 K 2024-12-04T20:29:03,804 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/.tmp/C/39879f8fdcf24a1f94d637dfda181ab0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/39879f8fdcf24a1f94d637dfda181ab0 2024-12-04T20:29:03,807 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/39879f8fdcf24a1f94d637dfda181ab0, entries=50, sequenceid=728, filesize=7.2 K 2024-12-04T20:29:03,808 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 2687aa8f5064ce602682f5c7138994be in 1267ms, sequenceid=728, compaction requested=true 2024-12-04T20:29:03,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/e257a7b29f004c71ab4a720095870f5e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/60d66a22555045e3a03ca85d79788700, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21adbc200940479f90fbb20dfcb4e7bd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d655e27423904899992402a7bb2b35c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/286afda85975430b9c82695992a51b55, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/cca8c0d863a646548c343a20b191bb32, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b05a7d3ecb3a4d118874566611fadbf2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5979f604388b4a14b18053d0f6661e67, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/48f86036eaa94e4c8b3c01a1e81b7fea, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/72299a5e72024dbc8039efe2d4c00b3e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/43d26d25ca0647b5a866fee6d780cb79, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6da640fb0fc54d829d8375b27e034571, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b47c1d49f2ec4309b76b6ea8e3a6a1b6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/f146241a55a44ea7ba5e78bbaf76657b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21824be7213642fbb4fc1bebd0b71add, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b8475403f80f4280ad9cd95bec43509d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6c9bb4a835f24ef1b352e16caae1dfeb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8266612e68d94736ac1ee6d1021ca15f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/c4619946d254446a8a133148795f4155, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/82f1fe18be6c4ef2b229c91ca7853b8f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5eeea1f4d8864bb1a9039a525ae9467c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6cf5a4673689413cb4b1d278d5cce271, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5a0c7124511a4dc79fbe4f8ee6a23bdb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d2f977eacb8744ce9edc6f7f15721df6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ab7585f70f8d436ebaa781ba21af5c95, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/1a253724215f4ebe80a3eb65412a2a6f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/33c3c34e2fb1449abff2dd49e84f51c1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/de131a76563740f5bfa5b537bae7a5ae, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5b6bdc9254a6487797b5b765d588c16e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/db6f109b5ebb4ec099978328dcf92fc2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6aa5fcb03f88445ba2f6dc25b82b9ccd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8a733d81441746fa814f844a4a957b22, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14] to archive 2024-12-04T20:29:03,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:29:03,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d3f69d7e8d9c49e48a6f3fd5bf7f2488 2024-12-04T20:29:03,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/e257a7b29f004c71ab4a720095870f5e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/e257a7b29f004c71ab4a720095870f5e 2024-12-04T20:29:03,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/60d66a22555045e3a03ca85d79788700 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/60d66a22555045e3a03ca85d79788700 2024-12-04T20:29:03,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21adbc200940479f90fbb20dfcb4e7bd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21adbc200940479f90fbb20dfcb4e7bd 2024-12-04T20:29:03,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d655e27423904899992402a7bb2b35c2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d655e27423904899992402a7bb2b35c2 2024-12-04T20:29:03,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/286afda85975430b9c82695992a51b55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/286afda85975430b9c82695992a51b55 2024-12-04T20:29:03,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/cca8c0d863a646548c343a20b191bb32 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/cca8c0d863a646548c343a20b191bb32 2024-12-04T20:29:03,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b05a7d3ecb3a4d118874566611fadbf2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b05a7d3ecb3a4d118874566611fadbf2 2024-12-04T20:29:03,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5979f604388b4a14b18053d0f6661e67 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5979f604388b4a14b18053d0f6661e67 2024-12-04T20:29:03,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/48f86036eaa94e4c8b3c01a1e81b7fea to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/48f86036eaa94e4c8b3c01a1e81b7fea 2024-12-04T20:29:03,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/72299a5e72024dbc8039efe2d4c00b3e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/72299a5e72024dbc8039efe2d4c00b3e 2024-12-04T20:29:03,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/43d26d25ca0647b5a866fee6d780cb79 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/43d26d25ca0647b5a866fee6d780cb79 2024-12-04T20:29:03,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6da640fb0fc54d829d8375b27e034571 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6da640fb0fc54d829d8375b27e034571 2024-12-04T20:29:03,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b47c1d49f2ec4309b76b6ea8e3a6a1b6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b47c1d49f2ec4309b76b6ea8e3a6a1b6 2024-12-04T20:29:03,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/f146241a55a44ea7ba5e78bbaf76657b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/f146241a55a44ea7ba5e78bbaf76657b 2024-12-04T20:29:03,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21824be7213642fbb4fc1bebd0b71add to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/21824be7213642fbb4fc1bebd0b71add 2024-12-04T20:29:03,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b8475403f80f4280ad9cd95bec43509d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/b8475403f80f4280ad9cd95bec43509d 2024-12-04T20:29:03,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6c9bb4a835f24ef1b352e16caae1dfeb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6c9bb4a835f24ef1b352e16caae1dfeb 2024-12-04T20:29:03,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8266612e68d94736ac1ee6d1021ca15f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8266612e68d94736ac1ee6d1021ca15f 2024-12-04T20:29:03,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/c4619946d254446a8a133148795f4155 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/c4619946d254446a8a133148795f4155 2024-12-04T20:29:03,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/82f1fe18be6c4ef2b229c91ca7853b8f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/82f1fe18be6c4ef2b229c91ca7853b8f 2024-12-04T20:29:03,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5eeea1f4d8864bb1a9039a525ae9467c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5eeea1f4d8864bb1a9039a525ae9467c 2024-12-04T20:29:03,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6cf5a4673689413cb4b1d278d5cce271 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6cf5a4673689413cb4b1d278d5cce271 2024-12-04T20:29:03,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5a0c7124511a4dc79fbe4f8ee6a23bdb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5a0c7124511a4dc79fbe4f8ee6a23bdb 2024-12-04T20:29:03,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d2f977eacb8744ce9edc6f7f15721df6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/d2f977eacb8744ce9edc6f7f15721df6 2024-12-04T20:29:03,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ab7585f70f8d436ebaa781ba21af5c95 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ab7585f70f8d436ebaa781ba21af5c95 2024-12-04T20:29:03,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/1a253724215f4ebe80a3eb65412a2a6f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/1a253724215f4ebe80a3eb65412a2a6f 2024-12-04T20:29:03,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/33c3c34e2fb1449abff2dd49e84f51c1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/33c3c34e2fb1449abff2dd49e84f51c1 2024-12-04T20:29:03,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/de131a76563740f5bfa5b537bae7a5ae to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/de131a76563740f5bfa5b537bae7a5ae 2024-12-04T20:29:03,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5b6bdc9254a6487797b5b765d588c16e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/5b6bdc9254a6487797b5b765d588c16e 2024-12-04T20:29:03,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/db6f109b5ebb4ec099978328dcf92fc2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/db6f109b5ebb4ec099978328dcf92fc2 2024-12-04T20:29:03,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6aa5fcb03f88445ba2f6dc25b82b9ccd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6aa5fcb03f88445ba2f6dc25b82b9ccd 2024-12-04T20:29:03,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8a733d81441746fa814f844a4a957b22 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/8a733d81441746fa814f844a4a957b22 2024-12-04T20:29:03,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/37af5aa2ab1b46b0bfcd29bcdbf8ab14 2024-12-04T20:29:03,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/4874b39f1fe24013b793050e1f6c7924, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/c4c9867f607c4a32a479682437026279, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/6d96d2fbfacb4788af30cf9c91d8a1a0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/40a8391af55244f6862501cfcf94bc7c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9ab6d367fc19437797d66aa97f38ca3b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b4daece59ce641d384df3aeafa306692, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/aa5a9f8f6bfb4ca3b25773aaf4eac863, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/74430663e8f34676bd945ca5b3fa0478, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/606cf469f3ed478cbab3abf457de807d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1c601ae6dfb849a5aca5c0ac3fdf32ad, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0bc9f703d90d43f0bbc4b8cb167c0c54, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5c363af1791242a79c6a755932da0d1e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b2586ffa58f44b0a9ba1ff9bc5316866, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3706932d91e4454181fa778178a1a191, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3b3a6e2ce5ad4867b18373dc1a7f83ae, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/95ad53e9ae5849c28d407d48778fbca0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/a056440075514294a4fd6aa959b59dc2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b0584eb66399490e843fad3e2d553456, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/ccfb902d9fdd4d0c94188575c37f6088, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9b09c683c47e42f19922c753c9bc90d5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8fa17675b23640a8ab1e0e8c69acc370, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/96a5c9c1f91f41d7a941afdea000e79b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/361fc676496c40999546dfc8bebfd1d6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e775cc7e20a442bf9a14b6086510f5ee, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0296b32b38c9460585ee0ec359c7645c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/469fa75eb4714bb3a99ecac8b9b4df92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/7149f16ccf544425913a9eb297803440, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9bf38c4d949b4c38ac33ba912903e827, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f8b616345b044349896172fdff601124, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/617f33f9e11047cda526cc1a56ed14f0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8dac819277474d34bb06b4794152cf4e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/217d0e6679b146debc30304aa9cc739b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e8a410bdd244457abebcd8cff825f3ad, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/739df2c7363b455090aa6637582ac665] to archive 2024-12-04T20:29:03,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:29:03,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/4874b39f1fe24013b793050e1f6c7924 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/4874b39f1fe24013b793050e1f6c7924 2024-12-04T20:29:03,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/c4c9867f607c4a32a479682437026279 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/c4c9867f607c4a32a479682437026279 2024-12-04T20:29:03,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/6d96d2fbfacb4788af30cf9c91d8a1a0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/6d96d2fbfacb4788af30cf9c91d8a1a0 2024-12-04T20:29:03,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/40a8391af55244f6862501cfcf94bc7c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/40a8391af55244f6862501cfcf94bc7c 2024-12-04T20:29:03,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9ab6d367fc19437797d66aa97f38ca3b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9ab6d367fc19437797d66aa97f38ca3b 2024-12-04T20:29:03,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b4daece59ce641d384df3aeafa306692 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b4daece59ce641d384df3aeafa306692 2024-12-04T20:29:03,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/aa5a9f8f6bfb4ca3b25773aaf4eac863 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/aa5a9f8f6bfb4ca3b25773aaf4eac863 2024-12-04T20:29:03,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/74430663e8f34676bd945ca5b3fa0478 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/74430663e8f34676bd945ca5b3fa0478 2024-12-04T20:29:03,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/606cf469f3ed478cbab3abf457de807d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/606cf469f3ed478cbab3abf457de807d 2024-12-04T20:29:03,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1c601ae6dfb849a5aca5c0ac3fdf32ad to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1c601ae6dfb849a5aca5c0ac3fdf32ad 2024-12-04T20:29:03,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0bc9f703d90d43f0bbc4b8cb167c0c54 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0bc9f703d90d43f0bbc4b8cb167c0c54 2024-12-04T20:29:03,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5c363af1791242a79c6a755932da0d1e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5c363af1791242a79c6a755932da0d1e 2024-12-04T20:29:03,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b2586ffa58f44b0a9ba1ff9bc5316866 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b2586ffa58f44b0a9ba1ff9bc5316866 2024-12-04T20:29:03,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3706932d91e4454181fa778178a1a191 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3706932d91e4454181fa778178a1a191 2024-12-04T20:29:03,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3b3a6e2ce5ad4867b18373dc1a7f83ae to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/3b3a6e2ce5ad4867b18373dc1a7f83ae 2024-12-04T20:29:03,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/95ad53e9ae5849c28d407d48778fbca0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/95ad53e9ae5849c28d407d48778fbca0 2024-12-04T20:29:03,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/a056440075514294a4fd6aa959b59dc2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/a056440075514294a4fd6aa959b59dc2 2024-12-04T20:29:03,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b0584eb66399490e843fad3e2d553456 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/b0584eb66399490e843fad3e2d553456 2024-12-04T20:29:03,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/ccfb902d9fdd4d0c94188575c37f6088 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/ccfb902d9fdd4d0c94188575c37f6088 2024-12-04T20:29:03,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9b09c683c47e42f19922c753c9bc90d5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9b09c683c47e42f19922c753c9bc90d5 2024-12-04T20:29:03,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8fa17675b23640a8ab1e0e8c69acc370 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8fa17675b23640a8ab1e0e8c69acc370 2024-12-04T20:29:03,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/96a5c9c1f91f41d7a941afdea000e79b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/96a5c9c1f91f41d7a941afdea000e79b 2024-12-04T20:29:03,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/361fc676496c40999546dfc8bebfd1d6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/361fc676496c40999546dfc8bebfd1d6 2024-12-04T20:29:03,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e775cc7e20a442bf9a14b6086510f5ee to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e775cc7e20a442bf9a14b6086510f5ee 2024-12-04T20:29:03,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0296b32b38c9460585ee0ec359c7645c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/0296b32b38c9460585ee0ec359c7645c 2024-12-04T20:29:03,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/469fa75eb4714bb3a99ecac8b9b4df92 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/469fa75eb4714bb3a99ecac8b9b4df92 2024-12-04T20:29:03,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/7149f16ccf544425913a9eb297803440 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/7149f16ccf544425913a9eb297803440 2024-12-04T20:29:03,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9bf38c4d949b4c38ac33ba912903e827 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/9bf38c4d949b4c38ac33ba912903e827 2024-12-04T20:29:03,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f8b616345b044349896172fdff601124 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f8b616345b044349896172fdff601124 2024-12-04T20:29:03,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/617f33f9e11047cda526cc1a56ed14f0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/617f33f9e11047cda526cc1a56ed14f0 2024-12-04T20:29:03,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8dac819277474d34bb06b4794152cf4e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/8dac819277474d34bb06b4794152cf4e 2024-12-04T20:29:03,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/217d0e6679b146debc30304aa9cc739b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/217d0e6679b146debc30304aa9cc739b 2024-12-04T20:29:03,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e8a410bdd244457abebcd8cff825f3ad to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/e8a410bdd244457abebcd8cff825f3ad 2024-12-04T20:29:03,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/739df2c7363b455090aa6637582ac665 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/739df2c7363b455090aa6637582ac665 2024-12-04T20:29:03,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/21c72121fb894daea4b3ab69e42e17cd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/da0a6577193845e2b3764d000c51173e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4dc99060e5a0416d9735adedd47c40df, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e52ae9fbba564f50b27ea0235db98a66, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b5b69e5439b3456eb7cee2bea715141e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1aa7a2b9808e4049bee093830b54345f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6aea62bee8c9465ab58f3dcfd0afbe81, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/770d7b09190d4238aa273cff6ed93592, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8f87f07ca05b460eb04f01248b57a61e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/22be1721d093445eb6876aef78a723ca, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8b5b4802f93440b683a6b3d0cc0b4d24, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/cdb2d6d940264ca6abd0d9f19587b9e9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1cd65ee603b340b2bff0be141cde937f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e5f2dfb6d2c743c4aa060626c7688ea6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b50c48bf3ef8491ca0f2930dc420d3ec, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6492a7d2ffd8418b850fb9bcd3a56927, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6ec0e7b60c9b45ec978e0530af355b36, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/0b64794ea87d4472a46027d52fb371fc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/62300422a2ea478786068baab2ff4bd9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/89d78c39d5dc4c04b7d1c3ecafae74c7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/f00db18568d44785af0aedd569c6d53c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1ca7b1eeed114395ae8dfd6ff1866c83, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dccb61b5a5f84537b6f4014ebc1a1ada, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dceaedcd0fd94b29af3b4a981b16bffb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c32501ae346443568415ca6dc41ba1ec, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/a7812ecbce6544e98199b40c43b7fd8f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/be435898ae4f474490b6f90d4f9190ae, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4ebde424c8f64b7eb7b0766e1c7dcee7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/13e14df27f13484db6d2adecc25daa7a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4d1060a36f98497887990f21f92039ed, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c2006e5428944725a91222fd62541340, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c05b1e26d4ab4be6a0fb81497d9b7d1f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/74267d05c7a54e77a75c83f32ff3caf1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/7ec59836fa96440d9bd2a323df6d827a] to archive 2024-12-04T20:29:03,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:29:03,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/21c72121fb894daea4b3ab69e42e17cd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/21c72121fb894daea4b3ab69e42e17cd 2024-12-04T20:29:03,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/da0a6577193845e2b3764d000c51173e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/da0a6577193845e2b3764d000c51173e 2024-12-04T20:29:03,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4dc99060e5a0416d9735adedd47c40df to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4dc99060e5a0416d9735adedd47c40df 2024-12-04T20:29:03,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e52ae9fbba564f50b27ea0235db98a66 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e52ae9fbba564f50b27ea0235db98a66 2024-12-04T20:29:03,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b5b69e5439b3456eb7cee2bea715141e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b5b69e5439b3456eb7cee2bea715141e 2024-12-04T20:29:03,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1aa7a2b9808e4049bee093830b54345f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1aa7a2b9808e4049bee093830b54345f 2024-12-04T20:29:03,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6aea62bee8c9465ab58f3dcfd0afbe81 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6aea62bee8c9465ab58f3dcfd0afbe81 2024-12-04T20:29:03,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/770d7b09190d4238aa273cff6ed93592 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/770d7b09190d4238aa273cff6ed93592 2024-12-04T20:29:03,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8f87f07ca05b460eb04f01248b57a61e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8f87f07ca05b460eb04f01248b57a61e 2024-12-04T20:29:03,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/22be1721d093445eb6876aef78a723ca to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/22be1721d093445eb6876aef78a723ca 2024-12-04T20:29:03,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8b5b4802f93440b683a6b3d0cc0b4d24 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/8b5b4802f93440b683a6b3d0cc0b4d24 2024-12-04T20:29:03,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/cdb2d6d940264ca6abd0d9f19587b9e9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/cdb2d6d940264ca6abd0d9f19587b9e9 2024-12-04T20:29:03,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1cd65ee603b340b2bff0be141cde937f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1cd65ee603b340b2bff0be141cde937f 2024-12-04T20:29:03,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e5f2dfb6d2c743c4aa060626c7688ea6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/e5f2dfb6d2c743c4aa060626c7688ea6 2024-12-04T20:29:03,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b50c48bf3ef8491ca0f2930dc420d3ec to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b50c48bf3ef8491ca0f2930dc420d3ec 2024-12-04T20:29:03,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6492a7d2ffd8418b850fb9bcd3a56927 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6492a7d2ffd8418b850fb9bcd3a56927 2024-12-04T20:29:03,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6ec0e7b60c9b45ec978e0530af355b36 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/6ec0e7b60c9b45ec978e0530af355b36 2024-12-04T20:29:03,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/0b64794ea87d4472a46027d52fb371fc to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/0b64794ea87d4472a46027d52fb371fc 2024-12-04T20:29:03,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/62300422a2ea478786068baab2ff4bd9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/62300422a2ea478786068baab2ff4bd9 2024-12-04T20:29:03,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/89d78c39d5dc4c04b7d1c3ecafae74c7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/89d78c39d5dc4c04b7d1c3ecafae74c7 2024-12-04T20:29:03,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/f00db18568d44785af0aedd569c6d53c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/f00db18568d44785af0aedd569c6d53c 2024-12-04T20:29:03,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1ca7b1eeed114395ae8dfd6ff1866c83 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/1ca7b1eeed114395ae8dfd6ff1866c83 2024-12-04T20:29:03,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dccb61b5a5f84537b6f4014ebc1a1ada to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dccb61b5a5f84537b6f4014ebc1a1ada 2024-12-04T20:29:03,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dceaedcd0fd94b29af3b4a981b16bffb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/dceaedcd0fd94b29af3b4a981b16bffb 2024-12-04T20:29:03,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c32501ae346443568415ca6dc41ba1ec to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c32501ae346443568415ca6dc41ba1ec 2024-12-04T20:29:03,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/a7812ecbce6544e98199b40c43b7fd8f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/a7812ecbce6544e98199b40c43b7fd8f 2024-12-04T20:29:03,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/be435898ae4f474490b6f90d4f9190ae to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/be435898ae4f474490b6f90d4f9190ae 2024-12-04T20:29:03,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4ebde424c8f64b7eb7b0766e1c7dcee7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4ebde424c8f64b7eb7b0766e1c7dcee7 2024-12-04T20:29:03,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/13e14df27f13484db6d2adecc25daa7a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/13e14df27f13484db6d2adecc25daa7a 2024-12-04T20:29:03,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4d1060a36f98497887990f21f92039ed to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/4d1060a36f98497887990f21f92039ed 2024-12-04T20:29:03,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c2006e5428944725a91222fd62541340 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c2006e5428944725a91222fd62541340 2024-12-04T20:29:03,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c05b1e26d4ab4be6a0fb81497d9b7d1f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/c05b1e26d4ab4be6a0fb81497d9b7d1f 2024-12-04T20:29:03,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/74267d05c7a54e77a75c83f32ff3caf1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/74267d05c7a54e77a75c83f32ff3caf1 2024-12-04T20:29:03,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/7ec59836fa96440d9bd2a323df6d827a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/7ec59836fa96440d9bd2a323df6d827a 2024-12-04T20:29:03,899 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/recovered.edits/731.seqid, newMaxSeqId=731, maxSeqId=1 2024-12-04T20:29:03,899 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. 2024-12-04T20:29:03,899 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 2687aa8f5064ce602682f5c7138994be: Waiting for close lock at 1733344141466Running coprocessor pre-close hooks at 1733344141466Disabling compacts and flushes for region at 1733344141466Disabling writes for close at 1733344142541 (+1075 ms)Obtaining lock to block concurrent updates at 1733344142541Preparing flush snapshotting stores in 2687aa8f5064ce602682f5c7138994be at 1733344142541Finished memstore snapshotting TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733344142541Flushing stores of TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be. at 1733344142542 (+1 ms)Flushing 2687aa8f5064ce602682f5c7138994be/A: creating writer at 1733344142542Flushing 2687aa8f5064ce602682f5c7138994be/A: appending metadata at 1733344142544 (+2 ms)Flushing 2687aa8f5064ce602682f5c7138994be/A: closing flushed file at 1733344142544Flushing 2687aa8f5064ce602682f5c7138994be/B: creating writer at 1733344142956 (+412 ms)Flushing 2687aa8f5064ce602682f5c7138994be/B: appending metadata at 1733344142959 (+3 ms)Flushing 2687aa8f5064ce602682f5c7138994be/B: closing flushed file at 1733344142959Flushing 2687aa8f5064ce602682f5c7138994be/C: creating writer at 1733344143376 (+417 ms)Flushing 2687aa8f5064ce602682f5c7138994be/C: appending metadata at 1733344143379 (+3 ms)Flushing 2687aa8f5064ce602682f5c7138994be/C: closing flushed file at 1733344143379Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fca6003: reopening flushed file at 1733344143794 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dcc0adf: reopening flushed file at 1733344143798 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34cf7a0e: reopening flushed file at 1733344143803 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 2687aa8f5064ce602682f5c7138994be in 1267ms, sequenceid=728, compaction requested=true at 1733344143808 (+5 ms)Writing region close event to WAL at 1733344143897 (+89 ms)Running coprocessor post-close hooks at 1733344143899 (+2 ms)Closed at 1733344143899 2024-12-04T20:29:03,901 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 2687aa8f5064ce602682f5c7138994be 2024-12-04T20:29:03,901 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=2687aa8f5064ce602682f5c7138994be, regionState=CLOSED 2024-12-04T20:29:03,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:29:03,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-04T20:29:03,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 2687aa8f5064ce602682f5c7138994be, server=c2ef38372881,45443,1733344054291 in 2.5890 sec 2024-12-04T20:29:03,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-04T20:29:03,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2687aa8f5064ce602682f5c7138994be, UNASSIGN in 2.5940 sec 2024-12-04T20:29:03,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-04T20:29:03,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5970 sec 2024-12-04T20:29:03,908 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344143908"}]},"ts":"1733344143908"} 2024-12-04T20:29:03,909 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T20:29:03,909 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T20:29:03,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6050 sec 2024-12-04T20:29:04,026 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T20:29:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T20:29:05,464 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:05,465 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T20:29:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:05,470 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-04T20:29:05,472 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:05,474 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be 2024-12-04T20:29:05,477 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/recovered.edits] 2024-12-04T20:29:05,480 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/01984945867f48c49202d83fa65c2bc0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/01984945867f48c49202d83fa65c2bc0 2024-12-04T20:29:05,481 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6dc5f427998d4e648c1796f09de1db66 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/6dc5f427998d4e648c1796f09de1db66 2024-12-04T20:29:05,482 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ac98fc1f33ef4a979ed7a02f1dc337b4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/A/ac98fc1f33ef4a979ed7a02f1dc337b4 2024-12-04T20:29:05,484 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1b8587dd1ad549e1aa4b15ff1e16087a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/1b8587dd1ad549e1aa4b15ff1e16087a 2024-12-04T20:29:05,485 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5cb374b5670b4518ac627bf75c83c9d5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/5cb374b5670b4518ac627bf75c83c9d5 2024-12-04T20:29:05,486 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f5d8eb04f2cb41909ac077f8709072ef to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/B/f5d8eb04f2cb41909ac077f8709072ef 2024-12-04T20:29:05,489 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/266913e961e94dffafe5f9b6b3d57179 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/266913e961e94dffafe5f9b6b3d57179 2024-12-04T20:29:05,490 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/39879f8fdcf24a1f94d637dfda181ab0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/39879f8fdcf24a1f94d637dfda181ab0 2024-12-04T20:29:05,492 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b1ce3e22b45c44e694ca29797d53eb18 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/C/b1ce3e22b45c44e694ca29797d53eb18 2024-12-04T20:29:05,495 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/recovered.edits/731.seqid to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be/recovered.edits/731.seqid 2024-12-04T20:29:05,496 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/2687aa8f5064ce602682f5c7138994be 2024-12-04T20:29:05,496 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T20:29:05,499 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:05,503 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T20:29:05,506 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T20:29:05,507 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:05,507 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T20:29:05,508 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733344145507"}]},"ts":"9223372036854775807"} 2024-12-04T20:29:05,510 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T20:29:05,510 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2687aa8f5064ce602682f5c7138994be, NAME => 'TestAcidGuarantees,,1733344117480.2687aa8f5064ce602682f5c7138994be.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T20:29:05,510 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T20:29:05,510 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733344145510"}]},"ts":"9223372036854775807"} 2024-12-04T20:29:05,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T20:29:05,512 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:05,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-12-04T20:29:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-04T20:29:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T20:29:05,585 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:05,592 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 248), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=649 (was 548) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4795 (was 3719) - AvailableMemoryMB LEAK? - 2024-12-04T20:29:05,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=649, ProcessCount=11, AvailableMemoryMB=4795 2024-12-04T20:29:05,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:29:05,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:29:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:05,602 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:29:05,602 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:05,603 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-04T20:29:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T20:29:05,603 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:29:05,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742186_1362 (size=1061) 2024-12-04T20:29:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T20:29:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T20:29:06,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:29:06,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742187_1363 (size=53) 2024-12-04T20:29:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T20:29:06,424 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:29:06,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 00a3709f8e7a19ee64c7444672f69f55, disabling compactions & flushes 2024-12-04T20:29:06,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. after waiting 0 ms 2024-12-04T20:29:06,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00a3709f8e7a19ee64c7444672f69f55: Waiting for close lock at 1733344146425Disabling compacts and flushes for region at 1733344146425Disabling writes for close at 1733344146425Writing region close event to WAL at 1733344146425Closed at 1733344146425 2024-12-04T20:29:06,428 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:29:06,429 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733344146428"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733344146428"}]},"ts":"1733344146428"} 2024-12-04T20:29:06,433 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:29:06,435 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:29:06,436 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344146435"}]},"ts":"1733344146435"} 2024-12-04T20:29:06,437 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T20:29:06,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, ASSIGN}] 2024-12-04T20:29:06,438 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, ASSIGN 2024-12-04T20:29:06,439 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, ASSIGN; state=OFFLINE, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=false 2024-12-04T20:29:06,589 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:06,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, ASSIGN because future has completed 2024-12-04T20:29:06,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:29:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T20:29:06,752 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,752 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:29:06,753 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,753 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:29:06,753 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,753 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,756 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,758 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:06,758 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a3709f8e7a19ee64c7444672f69f55 columnFamilyName A 2024-12-04T20:29:06,758 DEBUG [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:06,759 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(327): Store=00a3709f8e7a19ee64c7444672f69f55/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:06,759 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,760 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:06,761 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a3709f8e7a19ee64c7444672f69f55 columnFamilyName B 2024-12-04T20:29:06,761 DEBUG [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:06,761 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(327): Store=00a3709f8e7a19ee64c7444672f69f55/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:06,762 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,763 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:06,763 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a3709f8e7a19ee64c7444672f69f55 columnFamilyName C 2024-12-04T20:29:06,763 DEBUG [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:06,764 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(327): Store=00a3709f8e7a19ee64c7444672f69f55/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:06,764 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,764 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,765 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,766 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,767 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,767 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,768 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:29:06,769 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,772 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:29:06,772 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 00a3709f8e7a19ee64c7444672f69f55; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71942687, jitterRate=0.07202957570552826}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:29:06,772 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:06,773 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 00a3709f8e7a19ee64c7444672f69f55: Running coprocessor pre-open hook at 1733344146753Writing region info on filesystem at 1733344146753Initializing all the Stores at 1733344146755 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344146755Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344146755Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344146756 (+1 ms)Cleaning up temporary data from old regions at 1733344146767 (+11 ms)Running coprocessor post-open hooks at 1733344146772 (+5 ms)Region opened successfully at 1733344146773 (+1 ms) 2024-12-04T20:29:06,774 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., pid=87, masterSystemTime=1733344146746 2024-12-04T20:29:06,775 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,776 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:06,776 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:06,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:29:06,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-04T20:29:06,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 in 186 msec 2024-12-04T20:29:06,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-04T20:29:06,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, ASSIGN in 344 msec 2024-12-04T20:29:06,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:29:06,784 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344146784"}]},"ts":"1733344146784"} 2024-12-04T20:29:06,786 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T20:29:06,787 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:29:06,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-12-04T20:29:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T20:29:07,753 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:07,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:07,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:07,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:07,757 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2a80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:07,759 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3487a0bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:07,761 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:07,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:07,764 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:29:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,43507,1733344053664 2024-12-04T20:29:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a562a6a 2024-12-04T20:29:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:29:07,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:29:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:29:07,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:29:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:07,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742188_1364 (size=1097) 2024-12-04T20:29:08,180 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T20:29:08,180 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T20:29:08,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:29:08,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, REOPEN/MOVE}] 2024-12-04T20:29:08,187 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, REOPEN/MOVE 2024-12-04T20:29:08,188 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, REOPEN/MOVE because future has completed 2024-12-04T20:29:08,190 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:29:08,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:29:08,343 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,343 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:29:08,343 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 00a3709f8e7a19ee64c7444672f69f55, disabling compactions & flushes 2024-12-04T20:29:08,343 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,343 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,343 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. after waiting 0 ms 2024-12-04T20:29:08,343 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,346 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T20:29:08,347 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,347 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 00a3709f8e7a19ee64c7444672f69f55: Waiting for close lock at 1733344148343Running coprocessor pre-close hooks at 1733344148343Disabling compacts and flushes for region at 1733344148343Disabling writes for close at 1733344148343Writing region close event to WAL at 1733344148344 (+1 ms)Running coprocessor post-close hooks at 1733344148347 (+3 ms)Closed at 1733344148347 2024-12-04T20:29:08,347 WARN [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 00a3709f8e7a19ee64c7444672f69f55 to self. 2024-12-04T20:29:08,348 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,349 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=CLOSED 2024-12-04T20:29:08,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:29:08,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-04T20:29:08,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 in 161 msec 2024-12-04T20:29:08,353 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, REOPEN/MOVE; state=CLOSED, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=true 2024-12-04T20:29:08,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, REOPEN/MOVE because future has completed 2024-12-04T20:29:08,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:29:08,659 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,659 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:29:08,659 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,659 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:29:08,659 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,659 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,660 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,661 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:08,662 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a3709f8e7a19ee64c7444672f69f55 columnFamilyName A 2024-12-04T20:29:08,663 DEBUG [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:08,663 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(327): Store=00a3709f8e7a19ee64c7444672f69f55/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:08,664 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,664 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:08,665 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a3709f8e7a19ee64c7444672f69f55 columnFamilyName B 2024-12-04T20:29:08,665 DEBUG [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:08,665 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(327): Store=00a3709f8e7a19ee64c7444672f69f55/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:08,665 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,666 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:08,666 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a3709f8e7a19ee64c7444672f69f55 columnFamilyName C 2024-12-04T20:29:08,666 DEBUG [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:08,667 INFO [StoreOpener-00a3709f8e7a19ee64c7444672f69f55-1 {}] regionserver.HStore(327): Store=00a3709f8e7a19ee64c7444672f69f55/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:08,667 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,667 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,667 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,668 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,669 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,669 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,670 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:29:08,671 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,672 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 00a3709f8e7a19ee64c7444672f69f55; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64695015, jitterRate=-0.03596915304660797}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:29:08,672 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,672 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 00a3709f8e7a19ee64c7444672f69f55: Running coprocessor pre-open hook at 1733344148659Writing region info on filesystem at 1733344148659Initializing all the Stores at 1733344148660 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344148660Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344148660Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344148660Cleaning up temporary data from old regions at 1733344148669 (+9 ms)Running coprocessor post-open hooks at 1733344148672 (+3 ms)Region opened successfully at 1733344148672 2024-12-04T20:29:08,673 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., pid=92, masterSystemTime=1733344148656 2024-12-04T20:29:08,675 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,675 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,675 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=OPEN, openSeqNum=5, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:29:08,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-04T20:29:08,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 in 172 msec 2024-12-04T20:29:08,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-04T20:29:08,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, REOPEN/MOVE in 493 msec 2024-12-04T20:29:08,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-04T20:29:08,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-12-04T20:29:08,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-12-04T20:29:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-04T20:29:08,687 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:08,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acf579b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,691 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef24789, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,693 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@489ca0e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,694 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da745fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e0523be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,700 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@917c8be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,702 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1694180b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551f1ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,706 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12792bee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,708 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1611affb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e10251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,712 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbfc510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e06f847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,722 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c6589b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,724 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f01c669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@617db898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,728 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35c16a3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1107ce01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:08,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:08,732 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc5b8c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:08,733 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1fdfee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:08,734 DEBUG [Thread-1645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,734 DEBUG [Thread-1649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,741 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,742 DEBUG [Thread-1664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,745 DEBUG [Thread-1662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,746 DEBUG [Thread-1653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,747 DEBUG [Thread-1653 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,747 DEBUG [Thread-1653 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:08,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:29:08,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:08,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:08,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:08,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:08,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:08,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:08,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,757 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-04T20:29:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T20:29:08,759 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:08,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:08,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:08,765 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,767 DEBUG [Thread-1660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,779 DEBUG [Thread-1647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,779 DEBUG [Thread-1651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:08,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:29:08,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344148767, max timestamp=1733344148776Num uniques 859; 2024-12-04T20:29:08,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:08,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344148767, max timestamp=1733344148776Num uniques 859; 2024-12-04T20:29:08,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:08,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344148767, max timestamp=1733344148776Num uniques 859; 2024-12-04T20:29:08,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:08,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204927d5289276640c08ff8185134a1976d_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344148741/Put/seqid=0 2024-12-04T20:29:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742189_1365 (size=14594) 2024-12-04T20:29:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344208807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344208808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344208812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344208814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344208814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T20:29:08,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:08,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:08,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:08,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:08,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:08,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344208932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344208932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344208932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344208933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344208933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:09,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T20:29:09,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344209142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344209142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344209142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344209142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344209142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:09,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204927d5289276640c08ff8185134a1976d_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204927d5289276640c08ff8185134a1976d_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:09,208 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/2150321162594b0aa28055f622f2bd19, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:09,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/2150321162594b0aa28055f622f2bd19 is 175, key is test_row_0/A:col10/1733344148741/Put/seqid=0 2024-12-04T20:29:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742190_1366 (size=39549) 2024-12-04T20:29:09,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:09,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:09,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T20:29:09,433 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:29:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344209452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344209452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344209452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344209462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344209462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:09,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:09,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,618 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/2150321162594b0aa28055f622f2bd19 2024-12-04T20:29:09,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/a83393351b2e4e64a6a08a4c644678e1 is 50, key is test_row_0/B:col10/1733344148741/Put/seqid=0 2024-12-04T20:29:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742191_1367 (size=12001) 2024-12-04T20:29:09,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/a83393351b2e4e64a6a08a4c644678e1 2024-12-04T20:29:09,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/8b31e869ba3d41baa2f550ff4a378467 is 50, key is test_row_0/C:col10/1733344148741/Put/seqid=0 2024-12-04T20:29:09,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742192_1368 (size=12001) 2024-12-04T20:29:09,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T20:29:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344209962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344209962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344209971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344209971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344209971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:09,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:09,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:09,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:09,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:10,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/8b31e869ba3d41baa2f550ff4a378467 2024-12-04T20:29:10,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/2150321162594b0aa28055f622f2bd19 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/2150321162594b0aa28055f622f2bd19 2024-12-04T20:29:10,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/2150321162594b0aa28055f622f2bd19, entries=200, sequenceid=15, filesize=38.6 K 2024-12-04T20:29:10,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/a83393351b2e4e64a6a08a4c644678e1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a83393351b2e4e64a6a08a4c644678e1 2024-12-04T20:29:10,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a83393351b2e4e64a6a08a4c644678e1, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T20:29:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/8b31e869ba3d41baa2f550ff4a378467 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/8b31e869ba3d41baa2f550ff4a378467 2024-12-04T20:29:10,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/8b31e869ba3d41baa2f550ff4a378467, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T20:29:10,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 00a3709f8e7a19ee64c7444672f69f55 in 1344ms, sequenceid=15, compaction requested=false 2024-12-04T20:29:10,093 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T20:29:10,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:10,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T20:29:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:10,139 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-04T20:29:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204382e9f4191d646bfa69ddccd45e7d7d6_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344148812/Put/seqid=0 2024-12-04T20:29:10,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742193_1369 (size=12154) 2024-12-04T20:29:10,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:10,553 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204382e9f4191d646bfa69ddccd45e7d7d6_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204382e9f4191d646bfa69ddccd45e7d7d6_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:10,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/9539ee62d743480f9ca32022e43d7642, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:10,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/9539ee62d743480f9ca32022e43d7642 is 175, key is test_row_0/A:col10/1733344148812/Put/seqid=0 2024-12-04T20:29:10,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742194_1370 (size=30955) 2024-12-04T20:29:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T20:29:10,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/9539ee62d743480f9ca32022e43d7642 2024-12-04T20:29:10,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0dd05eae0fcd4031bff57d0f746d7fe5 is 50, key is test_row_0/B:col10/1733344148812/Put/seqid=0 2024-12-04T20:29:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742195_1371 (size=12001) 2024-12-04T20:29:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:10,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344210996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344210997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344210997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344210999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344211000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344211111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344211111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344211112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344211112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344211112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344211321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344211321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344211321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344211322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344211322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,370 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0dd05eae0fcd4031bff57d0f746d7fe5 2024-12-04T20:29:11,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c16be74ca94549a29f5b96ba2da54888 is 50, key is test_row_0/C:col10/1733344148812/Put/seqid=0 2024-12-04T20:29:11,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742196_1372 (size=12001) 2024-12-04T20:29:11,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344211631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344211631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344211631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344211631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344211642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:11,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c16be74ca94549a29f5b96ba2da54888 2024-12-04T20:29:11,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/9539ee62d743480f9ca32022e43d7642 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/9539ee62d743480f9ca32022e43d7642 2024-12-04T20:29:11,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/9539ee62d743480f9ca32022e43d7642, entries=150, sequenceid=51, filesize=30.2 K 2024-12-04T20:29:11,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0dd05eae0fcd4031bff57d0f746d7fe5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0dd05eae0fcd4031bff57d0f746d7fe5 2024-12-04T20:29:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0dd05eae0fcd4031bff57d0f746d7fe5, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T20:29:11,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c16be74ca94549a29f5b96ba2da54888 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c16be74ca94549a29f5b96ba2da54888 2024-12-04T20:29:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c16be74ca94549a29f5b96ba2da54888, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T20:29:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 00a3709f8e7a19ee64c7444672f69f55 in 1658ms, sequenceid=51, compaction requested=false 2024-12-04T20:29:11,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:11,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-04T20:29:11,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0390 sec 2024-12-04T20:29:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.0440 sec 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:29:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:12,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:12,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:12,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:12,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:12,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e9c3e8b4842c4cfeb810cf8357e36c27_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344152165/Put/seqid=0 2024-12-04T20:29:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742197_1373 (size=12154) 2024-12-04T20:29:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344212199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344212205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733344152192, max timestamp=1733344152206Num uniques 859; 2024-12-04T20:29:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733344152192, max timestamp=1733344152206Num uniques 859; 2024-12-04T20:29:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:12,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733344152192, max timestamp=1733344152206Num uniques 859; 2024-12-04T20:29:12,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344212243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344212252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344212253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344212312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344212322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344212362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344212371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344212371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344212521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344212542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:12,583 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e9c3e8b4842c4cfeb810cf8357e36c27_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e9c3e8b4842c4cfeb810cf8357e36c27_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:12,583 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/3015fa82eef34a1f984a4a46bae7822b, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:12,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/3015fa82eef34a1f984a4a46bae7822b is 175, key is test_row_0/A:col10/1733344152165/Put/seqid=0 2024-12-04T20:29:12,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742198_1374 (size=30951) 2024-12-04T20:29:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344212581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344212581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344212581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344212841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344212862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344212901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344212901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344212901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T20:29:12,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:12,922 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-04T20:29:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T20:29:12,924 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:12,925 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:12,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:12,988 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/3015fa82eef34a1f984a4a46bae7822b 2024-12-04T20:29:12,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/62c2d6f733004888bc58550667aa80fd is 50, key is test_row_0/B:col10/1733344152165/Put/seqid=0 2024-12-04T20:29:12,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742199_1375 (size=9657) 2024-12-04T20:29:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T20:29:13,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T20:29:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T20:29:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T20:29:13,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344213351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:13,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344213371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:13,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T20:29:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/62c2d6f733004888bc58550667aa80fd 2024-12-04T20:29:13,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e076bbfb64144146ae3bf72dae125117 is 50, key is test_row_0/C:col10/1733344152165/Put/seqid=0 2024-12-04T20:29:13,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742200_1376 (size=9657) 2024-12-04T20:29:13,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344213412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:13,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344213412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344213421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:13,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T20:29:13,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:13,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T20:29:13,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T20:29:13,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:13,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:13,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e076bbfb64144146ae3bf72dae125117 2024-12-04T20:29:13,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/3015fa82eef34a1f984a4a46bae7822b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3015fa82eef34a1f984a4a46bae7822b 2024-12-04T20:29:13,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3015fa82eef34a1f984a4a46bae7822b, entries=150, sequenceid=63, filesize=30.2 K 2024-12-04T20:29:13,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/62c2d6f733004888bc58550667aa80fd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/62c2d6f733004888bc58550667aa80fd 2024-12-04T20:29:13,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/62c2d6f733004888bc58550667aa80fd, entries=100, sequenceid=63, filesize=9.4 K 2024-12-04T20:29:13,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e076bbfb64144146ae3bf72dae125117 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e076bbfb64144146ae3bf72dae125117 2024-12-04T20:29:13,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e076bbfb64144146ae3bf72dae125117, entries=100, sequenceid=63, filesize=9.4 K 2024-12-04T20:29:13,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 00a3709f8e7a19ee64c7444672f69f55 in 1659ms, sequenceid=63, compaction requested=true 2024-12-04T20:29:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:13,825 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:13,825 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:13,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:13,835 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:13,835 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/B is initiating minor compaction (all files) 2024-12-04T20:29:13,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/A is initiating minor compaction (all files) 2024-12-04T20:29:13,835 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/B in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,835 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/A in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,835 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/2150321162594b0aa28055f622f2bd19, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/9539ee62d743480f9ca32022e43d7642, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3015fa82eef34a1f984a4a46bae7822b] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=99.1 K 2024-12-04T20:29:13,835 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a83393351b2e4e64a6a08a4c644678e1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0dd05eae0fcd4031bff57d0f746d7fe5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/62c2d6f733004888bc58550667aa80fd] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=32.9 K 2024-12-04T20:29:13,835 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,835 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/2150321162594b0aa28055f622f2bd19, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/9539ee62d743480f9ca32022e43d7642, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3015fa82eef34a1f984a4a46bae7822b] 2024-12-04T20:29:13,835 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a83393351b2e4e64a6a08a4c644678e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733344148741 2024-12-04T20:29:13,836 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2150321162594b0aa28055f622f2bd19, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733344148741 2024-12-04T20:29:13,836 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd05eae0fcd4031bff57d0f746d7fe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733344148807 2024-12-04T20:29:13,836 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9539ee62d743480f9ca32022e43d7642, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733344148807 2024-12-04T20:29:13,836 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c2d6f733004888bc58550667aa80fd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344150996 2024-12-04T20:29:13,836 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3015fa82eef34a1f984a4a46bae7822b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344150996 2024-12-04T20:29:13,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T20:29:13,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-12-04T20:29:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:13,841 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:13,842 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:13,843 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/8bff761991c94767903b3c49ec7e7ed7 is 50, key is test_row_0/B:col10/1733344152165/Put/seqid=0 2024-12-04T20:29:13,847 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204cecbfa8e01e44cdcb445d8d276347de8_00a3709f8e7a19ee64c7444672f69f55 store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:13,848 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204cecbfa8e01e44cdcb445d8d276347de8_00a3709f8e7a19ee64c7444672f69f55, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:13,848 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cecbfa8e01e44cdcb445d8d276347de8_00a3709f8e7a19ee64c7444672f69f55 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:13,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742201_1377 (size=12104) 2024-12-04T20:29:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040b8032d4069948f1abc49fdece05ee7a_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344152241/Put/seqid=0 2024-12-04T20:29:13,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742202_1378 (size=4469) 2024-12-04T20:29:13,861 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/8bff761991c94767903b3c49ec7e7ed7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/8bff761991c94767903b3c49ec7e7ed7 2024-12-04T20:29:13,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742203_1379 (size=12154) 2024-12-04T20:29:13,865 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/B of 00a3709f8e7a19ee64c7444672f69f55 into 8bff761991c94767903b3c49ec7e7ed7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:13,865 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:13,865 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/B, priority=13, startTime=1733344153825; duration=0sec 2024-12-04T20:29:13,865 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:13,865 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:B 2024-12-04T20:29:13,865 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:13,866 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:13,866 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/C is initiating minor compaction (all files) 2024-12-04T20:29:13,866 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/C in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:13,866 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/8b31e869ba3d41baa2f550ff4a378467, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c16be74ca94549a29f5b96ba2da54888, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e076bbfb64144146ae3bf72dae125117] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=32.9 K 2024-12-04T20:29:13,866 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b31e869ba3d41baa2f550ff4a378467, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733344148741 2024-12-04T20:29:13,866 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c16be74ca94549a29f5b96ba2da54888, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733344148807 2024-12-04T20:29:13,867 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e076bbfb64144146ae3bf72dae125117, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344150996 2024-12-04T20:29:13,871 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#C#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:13,871 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/30b0ef8ddd664441980704785ae0c081 is 50, key is test_row_0/C:col10/1733344152165/Put/seqid=0 2024-12-04T20:29:13,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742204_1380 (size=12104) 2024-12-04T20:29:14,026 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T20:29:14,026 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-04T20:29:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T20:29:14,260 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:14,260 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/c93839cf9bfb497592bca1a3ff997fcf is 175, key is test_row_0/A:col10/1733344152165/Put/seqid=0 2024-12-04T20:29:14,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:14,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742205_1381 (size=31165) 2024-12-04T20:29:14,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040b8032d4069948f1abc49fdece05ee7a_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040b8032d4069948f1abc49fdece05ee7a_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/81adb66445eb45a79eb891d433b8bc6e, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/81adb66445eb45a79eb891d433b8bc6e is 175, key is test_row_0/A:col10/1733344152241/Put/seqid=0 2024-12-04T20:29:14,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742206_1382 (size=30955) 2024-12-04T20:29:14,278 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/30b0ef8ddd664441980704785ae0c081 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/30b0ef8ddd664441980704785ae0c081 2024-12-04T20:29:14,282 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/C of 00a3709f8e7a19ee64c7444672f69f55 into 30b0ef8ddd664441980704785ae0c081(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:14,282 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:14,282 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/C, priority=13, startTime=1733344153825; duration=0sec 2024-12-04T20:29:14,282 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:14,282 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:C 2024-12-04T20:29:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:14,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:14,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344214403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344214408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344214431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344214431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,436 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:14,437 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344214441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,447 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344214521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344214521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,668 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/c93839cf9bfb497592bca1a3ff997fcf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/c93839cf9bfb497592bca1a3ff997fcf 2024-12-04T20:29:14,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/81adb66445eb45a79eb891d433b8bc6e 2024-12-04T20:29:14,672 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/A of 00a3709f8e7a19ee64c7444672f69f55 into c93839cf9bfb497592bca1a3ff997fcf(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:14,672 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:14,672 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/A, priority=13, startTime=1733344153825; duration=0sec 2024-12-04T20:29:14,672 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:14,672 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:A 2024-12-04T20:29:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0e6a3c3642944d998d6b13b5f0134a99 is 50, key is test_row_0/B:col10/1733344152241/Put/seqid=0 2024-12-04T20:29:14,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742207_1383 (size=12001) 2024-12-04T20:29:14,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0e6a3c3642944d998d6b13b5f0134a99 2024-12-04T20:29:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/6ed3a4c2ca7b47adaece658b7625e7db is 50, key is test_row_0/C:col10/1733344152241/Put/seqid=0 2024-12-04T20:29:14,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742208_1384 (size=12001) 2024-12-04T20:29:14,696 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/6ed3a4c2ca7b47adaece658b7625e7db 2024-12-04T20:29:14,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/81adb66445eb45a79eb891d433b8bc6e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/81adb66445eb45a79eb891d433b8bc6e 2024-12-04T20:29:14,703 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/81adb66445eb45a79eb891d433b8bc6e, entries=150, sequenceid=96, filesize=30.2 K 2024-12-04T20:29:14,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0e6a3c3642944d998d6b13b5f0134a99 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e6a3c3642944d998d6b13b5f0134a99 2024-12-04T20:29:14,708 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e6a3c3642944d998d6b13b5f0134a99, entries=150, sequenceid=96, filesize=11.7 K 2024-12-04T20:29:14,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/6ed3a4c2ca7b47adaece658b7625e7db as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6ed3a4c2ca7b47adaece658b7625e7db 2024-12-04T20:29:14,712 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6ed3a4c2ca7b47adaece658b7625e7db, entries=150, sequenceid=96, filesize=11.7 K 2024-12-04T20:29:14,712 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for 00a3709f8e7a19ee64c7444672f69f55 in 872ms, sequenceid=96, compaction requested=false 2024-12-04T20:29:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:14,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:14,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-04T20:29:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-04T20:29:14,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-04T20:29:14,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7890 sec 2024-12-04T20:29:14,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.7930 sec 2024-12-04T20:29:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:14,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:29:14,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:14,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:14,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:14,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:14,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:14,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:14,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d9ac2b26f3044111aaa97b3b1b74ca58_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344154396/Put/seqid=0 2024-12-04T20:29:14,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742209_1385 (size=17034) 2024-12-04T20:29:14,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344214781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344214785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344214891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344214892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T20:29:15,071 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:15,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-04T20:29:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T20:29:15,073 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:15,074 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:15,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:15,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344215101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:15,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344215101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:15,147 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:15,151 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d9ac2b26f3044111aaa97b3b1b74ca58_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d9ac2b26f3044111aaa97b3b1b74ca58_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:15,151 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/df0c4504fa534dfc8497993de5539727, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:15,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/df0c4504fa534dfc8497993de5539727 is 175, key is test_row_0/A:col10/1733344154396/Put/seqid=0 2024-12-04T20:29:15,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742210_1386 (size=48139) 2024-12-04T20:29:15,158 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/df0c4504fa534dfc8497993de5539727 2024-12-04T20:29:15,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/f0e79383acf348e682b476781cf7acad is 50, key is test_row_0/B:col10/1733344154396/Put/seqid=0 2024-12-04T20:29:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T20:29:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742211_1387 (size=12001) 2024-12-04T20:29:15,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T20:29:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T20:29:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T20:29:15,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344215411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344215411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:15,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T20:29:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:15,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/f0e79383acf348e682b476781cf7acad 2024-12-04T20:29:15,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e43e6d02520c42e2bc21f3b65d86bc8f is 50, key is test_row_0/C:col10/1733344154396/Put/seqid=0 2024-12-04T20:29:15,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742212_1388 (size=12001) 2024-12-04T20:29:15,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e43e6d02520c42e2bc21f3b65d86bc8f 2024-12-04T20:29:15,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/df0c4504fa534dfc8497993de5539727 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df0c4504fa534dfc8497993de5539727 2024-12-04T20:29:15,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df0c4504fa534dfc8497993de5539727, entries=250, sequenceid=113, filesize=47.0 K 2024-12-04T20:29:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/f0e79383acf348e682b476781cf7acad as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f0e79383acf348e682b476781cf7acad 2024-12-04T20:29:15,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f0e79383acf348e682b476781cf7acad, entries=150, sequenceid=113, filesize=11.7 K 2024-12-04T20:29:15,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e43e6d02520c42e2bc21f3b65d86bc8f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e43e6d02520c42e2bc21f3b65d86bc8f 2024-12-04T20:29:15,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e43e6d02520c42e2bc21f3b65d86bc8f, entries=150, sequenceid=113, filesize=11.7 K 2024-12-04T20:29:15,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 00a3709f8e7a19ee64c7444672f69f55 in 874ms, sequenceid=113, compaction requested=true 2024-12-04T20:29:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:15,609 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:15,609 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:15,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:15,610 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:15,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/A is initiating minor compaction (all files) 2024-12-04T20:29:15,610 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/B is initiating minor compaction (all files) 2024-12-04T20:29:15,610 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/B in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,610 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/A in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,610 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/8bff761991c94767903b3c49ec7e7ed7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e6a3c3642944d998d6b13b5f0134a99, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f0e79383acf348e682b476781cf7acad] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=35.3 K 2024-12-04T20:29:15,610 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/c93839cf9bfb497592bca1a3ff997fcf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/81adb66445eb45a79eb891d433b8bc6e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df0c4504fa534dfc8497993de5539727] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=107.7 K 2024-12-04T20:29:15,610 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/c93839cf9bfb497592bca1a3ff997fcf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/81adb66445eb45a79eb891d433b8bc6e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df0c4504fa534dfc8497993de5539727] 2024-12-04T20:29:15,610 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bff761991c94767903b3c49ec7e7ed7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344148813 2024-12-04T20:29:15,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c93839cf9bfb497592bca1a3ff997fcf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344148813 2024-12-04T20:29:15,611 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e6a3c3642944d998d6b13b5f0134a99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344152228 2024-12-04T20:29:15,611 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81adb66445eb45a79eb891d433b8bc6e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344152228 2024-12-04T20:29:15,611 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e79383acf348e682b476781cf7acad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733344154396 2024-12-04T20:29:15,611 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting df0c4504fa534dfc8497993de5539727, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733344154394 2024-12-04T20:29:15,615 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:15,615 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/15e64de3c8714664871378f558cfef54 is 50, key is test_row_0/B:col10/1733344154396/Put/seqid=0 2024-12-04T20:29:15,616 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:15,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742213_1389 (size=12207) 2024-12-04T20:29:15,620 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204516b5aea00a34719a4b81fa69bae4f56_00a3709f8e7a19ee64c7444672f69f55 store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:15,622 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/15e64de3c8714664871378f558cfef54 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/15e64de3c8714664871378f558cfef54 2024-12-04T20:29:15,622 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204516b5aea00a34719a4b81fa69bae4f56_00a3709f8e7a19ee64c7444672f69f55, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:15,622 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204516b5aea00a34719a4b81fa69bae4f56_00a3709f8e7a19ee64c7444672f69f55 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:15,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742214_1390 (size=4469) 2024-12-04T20:29:15,626 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/B of 00a3709f8e7a19ee64c7444672f69f55 into 15e64de3c8714664871378f558cfef54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:15,626 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:15,626 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/B, priority=13, startTime=1733344155609; duration=0sec 2024-12-04T20:29:15,626 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:15,626 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:B 2024-12-04T20:29:15,626 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:15,627 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#A#compaction#331 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:15,627 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:15,627 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/C is initiating minor compaction (all files) 2024-12-04T20:29:15,627 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/C in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,627 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/3271537e01b044ac89c37daef20e0786 is 175, key is test_row_0/A:col10/1733344154396/Put/seqid=0 2024-12-04T20:29:15,627 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/30b0ef8ddd664441980704785ae0c081, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6ed3a4c2ca7b47adaece658b7625e7db, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e43e6d02520c42e2bc21f3b65d86bc8f] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=35.3 K 2024-12-04T20:29:15,627 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b0ef8ddd664441980704785ae0c081, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733344148813 2024-12-04T20:29:15,628 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed3a4c2ca7b47adaece658b7625e7db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733344152228 2024-12-04T20:29:15,628 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e43e6d02520c42e2bc21f3b65d86bc8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733344154396 2024-12-04T20:29:15,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742215_1391 (size=31161) 2024-12-04T20:29:15,633 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:15,634 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/b4b19aec4e43484fad79b69ece8c6081 is 50, key is test_row_0/C:col10/1733344154396/Put/seqid=0 2024-12-04T20:29:15,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742216_1392 (size=12207) 2024-12-04T20:29:15,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T20:29:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:15,685 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T20:29:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046efbeb1950f34666a985da25da4df03d_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344154784/Put/seqid=0 2024-12-04T20:29:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742217_1393 (size=12254) 2024-12-04T20:29:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T20:29:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:15,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:15,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344215962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344215963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,033 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/3271537e01b044ac89c37daef20e0786 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3271537e01b044ac89c37daef20e0786 2024-12-04T20:29:16,038 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/A of 00a3709f8e7a19ee64c7444672f69f55 into 3271537e01b044ac89c37daef20e0786(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:16,038 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:16,038 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/A, priority=13, startTime=1733344155609; duration=0sec 2024-12-04T20:29:16,038 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:16,038 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:A 2024-12-04T20:29:16,040 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/b4b19aec4e43484fad79b69ece8c6081 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/b4b19aec4e43484fad79b69ece8c6081 2024-12-04T20:29:16,043 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/C of 00a3709f8e7a19ee64c7444672f69f55 into b4b19aec4e43484fad79b69ece8c6081(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:16,044 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:16,044 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/C, priority=13, startTime=1733344155609; duration=0sec 2024-12-04T20:29:16,044 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:16,044 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:C 2024-12-04T20:29:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344216081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344216082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:16,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046efbeb1950f34666a985da25da4df03d_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046efbeb1950f34666a985da25da4df03d_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/f2795b6c9aae40ada746f16b502ee2dc, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/f2795b6c9aae40ada746f16b502ee2dc is 175, key is test_row_0/A:col10/1733344154784/Put/seqid=0 2024-12-04T20:29:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742218_1394 (size=31055) 2024-12-04T20:29:16,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/f2795b6c9aae40ada746f16b502ee2dc 2024-12-04T20:29:16,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/fc8d69cfec524ebdbffa9f0300e239bd is 50, key is test_row_0/B:col10/1733344154784/Put/seqid=0 2024-12-04T20:29:16,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742219_1395 (size=12101) 2024-12-04T20:29:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T20:29:16,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344216291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344216291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344216442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,446 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344216461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,464 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344216471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,476 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,512 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/fc8d69cfec524ebdbffa9f0300e239bd 2024-12-04T20:29:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/41e952596eab436bbd971ee1fa3d0081 is 50, key is test_row_0/C:col10/1733344154784/Put/seqid=0 2024-12-04T20:29:16,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742220_1396 (size=12101) 2024-12-04T20:29:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344216611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344216611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:16,920 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/41e952596eab436bbd971ee1fa3d0081 2024-12-04T20:29:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/f2795b6c9aae40ada746f16b502ee2dc as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2795b6c9aae40ada746f16b502ee2dc 2024-12-04T20:29:16,927 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2795b6c9aae40ada746f16b502ee2dc, entries=150, sequenceid=132, filesize=30.3 K 2024-12-04T20:29:16,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/fc8d69cfec524ebdbffa9f0300e239bd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/fc8d69cfec524ebdbffa9f0300e239bd 2024-12-04T20:29:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/fc8d69cfec524ebdbffa9f0300e239bd, entries=150, sequenceid=132, filesize=11.8 K 2024-12-04T20:29:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/41e952596eab436bbd971ee1fa3d0081 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/41e952596eab436bbd971ee1fa3d0081 2024-12-04T20:29:16,934 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/41e952596eab436bbd971ee1fa3d0081, entries=150, sequenceid=132, filesize=11.8 K 2024-12-04T20:29:16,935 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 00a3709f8e7a19ee64c7444672f69f55 in 1250ms, sequenceid=132, compaction requested=false 2024-12-04T20:29:16,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:16,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:16,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-04T20:29:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-04T20:29:16,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-04T20:29:16,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-12-04T20:29:16,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8680 sec 2024-12-04T20:29:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:17,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T20:29:17,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:17,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:17,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:17,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:17,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:17,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:17,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120438f16edcab124e66a6a19c3fa72698c0_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344155954/Put/seqid=0 2024-12-04T20:29:17,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742221_1397 (size=12304) 2024-12-04T20:29:17,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:17,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120438f16edcab124e66a6a19c3fa72698c0_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120438f16edcab124e66a6a19c3fa72698c0_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:17,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b8e883d902804e369f42ceacb50cd178, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:17,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b8e883d902804e369f42ceacb50cd178 is 175, key is test_row_0/A:col10/1733344155954/Put/seqid=0 2024-12-04T20:29:17,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742222_1398 (size=31105) 2024-12-04T20:29:17,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344217163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344217170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T20:29:17,221 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:17,222 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-04T20:29:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T20:29:17,223 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:17,224 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:17,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:17,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344217282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344217282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T20:29:17,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344217491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344217491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T20:29:17,544 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b8e883d902804e369f42ceacb50cd178 2024-12-04T20:29:17,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/2e44cb7122b24eb2a2dce6e29fdbd462 is 50, key is test_row_0/B:col10/1733344155954/Put/seqid=0 2024-12-04T20:29:17,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742223_1399 (size=12151) 2024-12-04T20:29:17,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344217801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344217801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:17,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T20:29:17,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/2e44cb7122b24eb2a2dce6e29fdbd462 2024-12-04T20:29:17,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/68f78469a2be4d3881893d3153c6ab83 is 50, key is test_row_0/C:col10/1733344155954/Put/seqid=0 2024-12-04T20:29:17,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742224_1400 (size=12151) 2024-12-04T20:29:17,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:17,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:18,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:18,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:18,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:18,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344218311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:18,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344218311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:18,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/68f78469a2be4d3881893d3153c6ab83 2024-12-04T20:29:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T20:29:18,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b8e883d902804e369f42ceacb50cd178 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b8e883d902804e369f42ceacb50cd178 2024-12-04T20:29:18,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b8e883d902804e369f42ceacb50cd178, entries=150, sequenceid=149, filesize=30.4 K 2024-12-04T20:29:18,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/2e44cb7122b24eb2a2dce6e29fdbd462 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2e44cb7122b24eb2a2dce6e29fdbd462 2024-12-04T20:29:18,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2e44cb7122b24eb2a2dce6e29fdbd462, entries=150, sequenceid=149, filesize=11.9 K 2024-12-04T20:29:18,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/68f78469a2be4d3881893d3153c6ab83 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/68f78469a2be4d3881893d3153c6ab83 2024-12-04T20:29:18,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/68f78469a2be4d3881893d3153c6ab83, entries=150, sequenceid=149, filesize=11.9 K 2024-12-04T20:29:18,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 00a3709f8e7a19ee64c7444672f69f55 in 1249ms, sequenceid=149, compaction requested=true 2024-12-04T20:29:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:18,375 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:18,375 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:18,375 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:18,375 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:18,375 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/A is initiating minor compaction (all files) 2024-12-04T20:29:18,375 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/B is initiating minor compaction (all files) 2024-12-04T20:29:18,376 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/B in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,376 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/A in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,376 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/15e64de3c8714664871378f558cfef54, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/fc8d69cfec524ebdbffa9f0300e239bd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2e44cb7122b24eb2a2dce6e29fdbd462] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=35.6 K 2024-12-04T20:29:18,376 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3271537e01b044ac89c37daef20e0786, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2795b6c9aae40ada746f16b502ee2dc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b8e883d902804e369f42ceacb50cd178] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=91.1 K 2024-12-04T20:29:18,376 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,376 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3271537e01b044ac89c37daef20e0786, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2795b6c9aae40ada746f16b502ee2dc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b8e883d902804e369f42ceacb50cd178] 2024-12-04T20:29:18,376 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 15e64de3c8714664871378f558cfef54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733344154396 2024-12-04T20:29:18,376 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3271537e01b044ac89c37daef20e0786, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733344154396 2024-12-04T20:29:18,376 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting fc8d69cfec524ebdbffa9f0300e239bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733344154777 2024-12-04T20:29:18,376 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2795b6c9aae40ada746f16b502ee2dc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733344154777 2024-12-04T20:29:18,376 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e44cb7122b24eb2a2dce6e29fdbd462, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733344155928 2024-12-04T20:29:18,377 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8e883d902804e369f42ceacb50cd178, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733344155928 2024-12-04T20:29:18,381 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:18,382 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:18,382 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204febd46dc31b44f28813e26557f69e86b_00a3709f8e7a19ee64c7444672f69f55 store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:18,383 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/21417e2ac3634efa8c1cb9e192b15741 is 50, key is test_row_0/B:col10/1733344155954/Put/seqid=0 2024-12-04T20:29:18,384 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204febd46dc31b44f28813e26557f69e86b_00a3709f8e7a19ee64c7444672f69f55, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:18,384 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204febd46dc31b44f28813e26557f69e86b_00a3709f8e7a19ee64c7444672f69f55 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:18,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742225_1401 (size=12459) 2024-12-04T20:29:18,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742226_1402 (size=4469) 2024-12-04T20:29:18,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T20:29:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T20:29:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:18,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120432b566a5254049c1a6efa9a2289d6946_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344157148/Put/seqid=0 2024-12-04T20:29:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742227_1403 (size=12304) 2024-12-04T20:29:18,791 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:18,792 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/a6f0f6a63b18497db41ebef32f37bcce is 175, key is test_row_0/A:col10/1733344155954/Put/seqid=0 2024-12-04T20:29:18,798 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/21417e2ac3634efa8c1cb9e192b15741 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/21417e2ac3634efa8c1cb9e192b15741 2024-12-04T20:29:18,803 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/B of 00a3709f8e7a19ee64c7444672f69f55 into 21417e2ac3634efa8c1cb9e192b15741(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:18,803 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:18,803 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/B, priority=13, startTime=1733344158375; duration=0sec 2024-12-04T20:29:18,803 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:18,803 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:B 2024-12-04T20:29:18,803 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:18,805 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:18,805 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/C is initiating minor compaction (all files) 2024-12-04T20:29:18,805 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/C in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:18,805 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/b4b19aec4e43484fad79b69ece8c6081, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/41e952596eab436bbd971ee1fa3d0081, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/68f78469a2be4d3881893d3153c6ab83] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=35.6 K 2024-12-04T20:29:18,805 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b19aec4e43484fad79b69ece8c6081, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733344154396 2024-12-04T20:29:18,806 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 41e952596eab436bbd971ee1fa3d0081, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733344154777 2024-12-04T20:29:18,806 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f78469a2be4d3881893d3153c6ab83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733344155928 2024-12-04T20:29:18,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742228_1404 (size=31413) 2024-12-04T20:29:18,814 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/a6f0f6a63b18497db41ebef32f37bcce as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/a6f0f6a63b18497db41ebef32f37bcce 2024-12-04T20:29:18,821 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#C#compaction#342 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:18,822 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/9dd2ad2e2db843c8870b5cff29b11999 is 50, key is test_row_0/C:col10/1733344155954/Put/seqid=0 2024-12-04T20:29:18,823 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/A of 00a3709f8e7a19ee64c7444672f69f55 into a6f0f6a63b18497db41ebef32f37bcce(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:18,823 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:18,823 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/A, priority=13, startTime=1733344158375; duration=0sec 2024-12-04T20:29:18,823 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:18,823 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:A 2024-12-04T20:29:18,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742229_1405 (size=12459) 2024-12-04T20:29:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:18,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120432b566a5254049c1a6efa9a2289d6946_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120432b566a5254049c1a6efa9a2289d6946_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:18,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/642e0ea8954d4e89902465a3a2c25c59, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:18,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/642e0ea8954d4e89902465a3a2c25c59 is 175, key is test_row_0/A:col10/1733344157148/Put/seqid=0 2024-12-04T20:29:18,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742230_1406 (size=31105) 2024-12-04T20:29:18,875 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/642e0ea8954d4e89902465a3a2c25c59 2024-12-04T20:29:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/da51d8bc80284bbb90840f666a4a9e14 is 50, key is test_row_0/B:col10/1733344157148/Put/seqid=0 2024-12-04T20:29:18,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742231_1407 (size=12151) 2024-12-04T20:29:18,893 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/da51d8bc80284bbb90840f666a4a9e14 2024-12-04T20:29:18,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/0a8e5ebc3cdc4d048e2e2d1898050c32 is 50, key is test_row_0/C:col10/1733344157148/Put/seqid=0 2024-12-04T20:29:18,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742232_1408 (size=12151) 2024-12-04T20:29:19,232 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/9dd2ad2e2db843c8870b5cff29b11999 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/9dd2ad2e2db843c8870b5cff29b11999 2024-12-04T20:29:19,237 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/C of 00a3709f8e7a19ee64c7444672f69f55 into 9dd2ad2e2db843c8870b5cff29b11999(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:19,237 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:19,238 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/C, priority=13, startTime=1733344158375; duration=0sec 2024-12-04T20:29:19,238 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:19,238 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:C 2024-12-04T20:29:19,313 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/0a8e5ebc3cdc4d048e2e2d1898050c32 2024-12-04T20:29:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/642e0ea8954d4e89902465a3a2c25c59 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/642e0ea8954d4e89902465a3a2c25c59 2024-12-04T20:29:19,320 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/642e0ea8954d4e89902465a3a2c25c59, entries=150, sequenceid=166, filesize=30.4 K 2024-12-04T20:29:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/da51d8bc80284bbb90840f666a4a9e14 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/da51d8bc80284bbb90840f666a4a9e14 2024-12-04T20:29:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/da51d8bc80284bbb90840f666a4a9e14, entries=150, sequenceid=166, filesize=11.9 K 2024-12-04T20:29:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/0a8e5ebc3cdc4d048e2e2d1898050c32 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/0a8e5ebc3cdc4d048e2e2d1898050c32 2024-12-04T20:29:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/0a8e5ebc3cdc4d048e2e2d1898050c32, entries=150, sequenceid=166, filesize=11.9 K 2024-12-04T20:29:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,330 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 00a3709f8e7a19ee64c7444672f69f55 in 886ms, sequenceid=166, compaction requested=false 2024-12-04T20:29:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:19,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-04T20:29:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-04T20:29:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-04T20:29:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1080 sec 2024-12-04T20:29:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1120 sec 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T20:29:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,372 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,372 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-04T20:29:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,374 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T20:29:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,374 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T20:29:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:19,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:29:19,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:19,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:19,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:19,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:19,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:19,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:19,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120438d662193d684ddabcdd242212bae672_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344159438/Put/seqid=0 2024-12-04T20:29:19,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742233_1409 (size=14794) 2024-12-04T20:29:19,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T20:29:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344159534, max timestamp=1733344159561Num uniques 859; 2024-12-04T20:29:19,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344159534, max timestamp=1733344159561Num uniques 859; 2024-12-04T20:29:19,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:19,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:19,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344159534, max timestamp=1733344159561Num uniques 859; 2024-12-04T20:29:19,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:19,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344219631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:19,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344219633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:19,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T20:29:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T20:29:19,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344219752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:19,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344219752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:19,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T20:29:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,899 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:19,904 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120438d662193d684ddabcdd242212bae672_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120438d662193d684ddabcdd242212bae672_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:19,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/26a74ad7b0564b46ae18aeed2f988b8d, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:19,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/26a74ad7b0564b46ae18aeed2f988b8d is 175, key is test_row_0/A:col10/1733344159438/Put/seqid=0 2024-12-04T20:29:19,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742234_1410 (size=39749) 2024-12-04T20:29:19,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/26a74ad7b0564b46ae18aeed2f988b8d 2024-12-04T20:29:19,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/a42c473793c44ef8adeaba602517cc2d is 50, key is test_row_0/B:col10/1733344159438/Put/seqid=0 2024-12-04T20:29:19,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742235_1411 (size=12151) 2024-12-04T20:29:19,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344219971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:19,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344219972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:19,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T20:29:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:19,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T20:29:20,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T20:29:20,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:20,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:20,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:20,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T20:29:20,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:20,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:20,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344220292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344220292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/a42c473793c44ef8adeaba602517cc2d 2024-12-04T20:29:20,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/204718b76af346d7971d1e0bfaf4f03a is 50, key is test_row_0/C:col10/1733344159438/Put/seqid=0 2024-12-04T20:29:20,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742236_1412 (size=12151) 2024-12-04T20:29:20,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/204718b76af346d7971d1e0bfaf4f03a 2024-12-04T20:29:20,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/26a74ad7b0564b46ae18aeed2f988b8d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/26a74ad7b0564b46ae18aeed2f988b8d 2024-12-04T20:29:20,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/26a74ad7b0564b46ae18aeed2f988b8d, entries=200, sequenceid=179, filesize=38.8 K 2024-12-04T20:29:20,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/a42c473793c44ef8adeaba602517cc2d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a42c473793c44ef8adeaba602517cc2d 2024-12-04T20:29:20,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a42c473793c44ef8adeaba602517cc2d, entries=150, sequenceid=179, filesize=11.9 K 2024-12-04T20:29:20,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/204718b76af346d7971d1e0bfaf4f03a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/204718b76af346d7971d1e0bfaf4f03a 2024-12-04T20:29:20,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/204718b76af346d7971d1e0bfaf4f03a, entries=150, sequenceid=179, filesize=11.9 K 2024-12-04T20:29:20,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00a3709f8e7a19ee64c7444672f69f55 in 919ms, sequenceid=179, compaction requested=true 2024-12-04T20:29:20,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:20,405 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:20,406 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:20,406 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/A is initiating minor compaction (all files) 2024-12-04T20:29:20,406 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/A in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,406 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/a6f0f6a63b18497db41ebef32f37bcce, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/642e0ea8954d4e89902465a3a2c25c59, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/26a74ad7b0564b46ae18aeed2f988b8d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=99.9 K 2024-12-04T20:29:20,406 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,406 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/a6f0f6a63b18497db41ebef32f37bcce, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/642e0ea8954d4e89902465a3a2c25c59, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/26a74ad7b0564b46ae18aeed2f988b8d] 2024-12-04T20:29:20,407 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f0f6a63b18497db41ebef32f37bcce, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733344155928 2024-12-04T20:29:20,407 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 642e0ea8954d4e89902465a3a2c25c59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733344157148 2024-12-04T20:29:20,407 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26a74ad7b0564b46ae18aeed2f988b8d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733344159438 2024-12-04T20:29:20,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:20,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:20,415 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:20,416 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:20,416 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/B is initiating minor compaction (all files) 2024-12-04T20:29:20,416 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/B in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,416 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/21417e2ac3634efa8c1cb9e192b15741, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/da51d8bc80284bbb90840f666a4a9e14, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a42c473793c44ef8adeaba602517cc2d] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=35.9 K 2024-12-04T20:29:20,417 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 21417e2ac3634efa8c1cb9e192b15741, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733344155928 2024-12-04T20:29:20,417 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting da51d8bc80284bbb90840f666a4a9e14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733344157148 2024-12-04T20:29:20,417 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a42c473793c44ef8adeaba602517cc2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733344159438 2024-12-04T20:29:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:20,431 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:20,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T20:29:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,449 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T20:29:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:20,451 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:20,451 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/22e376cc5a5a4b26b1aff7f27712c5fe is 50, key is test_row_0/B:col10/1733344159438/Put/seqid=0 2024-12-04T20:29:20,462 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204bde7490303864fcf8e4832b38af30165_00a3709f8e7a19ee64c7444672f69f55 store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:20,464 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204bde7490303864fcf8e4832b38af30165_00a3709f8e7a19ee64c7444672f69f55, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:20,464 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204bde7490303864fcf8e4832b38af30165_00a3709f8e7a19ee64c7444672f69f55 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:20,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T20:29:20,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742237_1413 (size=12561) 2024-12-04T20:29:20,519 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/22e376cc5a5a4b26b1aff7f27712c5fe as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/22e376cc5a5a4b26b1aff7f27712c5fe 2024-12-04T20:29:20,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742238_1414 (size=4469) 2024-12-04T20:29:20,524 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/B of 00a3709f8e7a19ee64c7444672f69f55 into 22e376cc5a5a4b26b1aff7f27712c5fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:20,524 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:20,524 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/B, priority=13, startTime=1733344160415; duration=0sec 2024-12-04T20:29:20,524 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:20,524 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:B 2024-12-04T20:29:20,524 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:20,525 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:20,525 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/C is initiating minor compaction (all files) 2024-12-04T20:29:20,526 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/C in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:20,526 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/9dd2ad2e2db843c8870b5cff29b11999, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/0a8e5ebc3cdc4d048e2e2d1898050c32, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/204718b76af346d7971d1e0bfaf4f03a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=35.9 K 2024-12-04T20:29:20,526 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dd2ad2e2db843c8870b5cff29b11999, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733344155928 2024-12-04T20:29:20,526 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8e5ebc3cdc4d048e2e2d1898050c32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733344157148 2024-12-04T20:29:20,527 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 204718b76af346d7971d1e0bfaf4f03a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733344159438 2024-12-04T20:29:20,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e08c5cb84e454482b0796408becc2841_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344159631/Put/seqid=0 2024-12-04T20:29:20,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344220520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344220528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344220533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,555 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#C#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:20,556 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/d929437b854f4932ac59895cb5b3b371 is 50, key is test_row_0/C:col10/1733344159438/Put/seqid=0 2024-12-04T20:29:20,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742239_1415 (size=12304) 2024-12-04T20:29:20,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742240_1416 (size=12561) 2024-12-04T20:29:20,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344220641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344220652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344220663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344220811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344220811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344220852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344220871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344220882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:20,924 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#A#compaction#348 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:20,924 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/7aea34faf23f45d882729b527fedc901 is 175, key is test_row_0/A:col10/1733344159438/Put/seqid=0 2024-12-04T20:29:20,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742241_1417 (size=31515) 2024-12-04T20:29:20,960 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/7aea34faf23f45d882729b527fedc901 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/7aea34faf23f45d882729b527fedc901 2024-12-04T20:29:20,968 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/A of 00a3709f8e7a19ee64c7444672f69f55 into 7aea34faf23f45d882729b527fedc901(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:20,968 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:20,968 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/A, priority=13, startTime=1733344160405; duration=0sec 2024-12-04T20:29:20,968 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:20,968 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:A 2024-12-04T20:29:21,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:21,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e08c5cb84e454482b0796408becc2841_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e08c5cb84e454482b0796408becc2841_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/295ba1a038a04d6f892044c12819e51c, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/295ba1a038a04d6f892044c12819e51c is 175, key is test_row_0/A:col10/1733344159631/Put/seqid=0 2024-12-04T20:29:21,019 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/d929437b854f4932ac59895cb5b3b371 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d929437b854f4932ac59895cb5b3b371 2024-12-04T20:29:21,028 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/C of 00a3709f8e7a19ee64c7444672f69f55 into d929437b854f4932ac59895cb5b3b371(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:21,028 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:21,028 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/C, priority=13, startTime=1733344160422; duration=0sec 2024-12-04T20:29:21,028 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:21,028 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:C 2024-12-04T20:29:21,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742242_1418 (size=31105) 2024-12-04T20:29:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344221161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344221191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344221201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/295ba1a038a04d6f892044c12819e51c 2024-12-04T20:29:21,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/551009bafaf64ce7acd6de9a1e042d92 is 50, key is test_row_0/B:col10/1733344159631/Put/seqid=0 2024-12-04T20:29:21,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742243_1419 (size=12151) 2024-12-04T20:29:21,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/551009bafaf64ce7acd6de9a1e042d92 2024-12-04T20:29:21,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/6d4fd101fdc243c1a1da0ae2074a8347 is 50, key is test_row_0/C:col10/1733344159631/Put/seqid=0 2024-12-04T20:29:21,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742244_1420 (size=12151) 2024-12-04T20:29:21,517 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/6d4fd101fdc243c1a1da0ae2074a8347 2024-12-04T20:29:21,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/295ba1a038a04d6f892044c12819e51c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/295ba1a038a04d6f892044c12819e51c 2024-12-04T20:29:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T20:29:21,525 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/295ba1a038a04d6f892044c12819e51c, entries=150, sequenceid=215, filesize=30.4 K 2024-12-04T20:29:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/551009bafaf64ce7acd6de9a1e042d92 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/551009bafaf64ce7acd6de9a1e042d92 2024-12-04T20:29:21,534 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/551009bafaf64ce7acd6de9a1e042d92, entries=150, sequenceid=215, filesize=11.9 K 2024-12-04T20:29:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/6d4fd101fdc243c1a1da0ae2074a8347 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6d4fd101fdc243c1a1da0ae2074a8347 2024-12-04T20:29:21,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6d4fd101fdc243c1a1da0ae2074a8347, entries=150, sequenceid=215, filesize=11.9 K 2024-12-04T20:29:21,541 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00a3709f8e7a19ee64c7444672f69f55 in 1092ms, sequenceid=215, compaction requested=false 2024-12-04T20:29:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:21,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-04T20:29:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-04T20:29:21,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-04T20:29:21,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-12-04T20:29:21,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1730 sec 2024-12-04T20:29:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:21,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T20:29:21,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:21,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:21,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:21,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:21,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:21,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:21,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047bbab141772244c8ba0c5cf66b8f2442_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344160509/Put/seqid=0 2024-12-04T20:29:21,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742245_1421 (size=14794) 2024-12-04T20:29:21,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344221734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344161722, max timestamp=1733344161740Num uniques 859; 2024-12-04T20:29:21,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:21,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344161722, max timestamp=1733344161740Num uniques 859; 2024-12-04T20:29:21,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:21,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344161722, max timestamp=1733344161740Num uniques 859; 2024-12-04T20:29:21,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344221834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344221835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344221841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344221841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,849 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,850 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344221851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344221941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344221951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:22,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344222071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:22,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:22,109 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047bbab141772244c8ba0c5cf66b8f2442_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047bbab141772244c8ba0c5cf66b8f2442_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:22,110 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e673f69444c54a09a8c28528906e8f00, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:22,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e673f69444c54a09a8c28528906e8f00 is 175, key is test_row_0/A:col10/1733344160509/Put/seqid=0 2024-12-04T20:29:22,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742246_1422 (size=39749) 2024-12-04T20:29:22,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e673f69444c54a09a8c28528906e8f00 2024-12-04T20:29:22,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/2f9973510fc947d4953f0fba88e7d58a is 50, key is test_row_0/B:col10/1733344160509/Put/seqid=0 2024-12-04T20:29:22,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344222151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:22,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742247_1423 (size=12151) 2024-12-04T20:29:22,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344222173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:22,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344222381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:22,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344222462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:22,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:22,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344222491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:22,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/2f9973510fc947d4953f0fba88e7d58a 2024-12-04T20:29:22,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/3d2556e5fe1b4e3baf6228dbd79d353f is 50, key is test_row_0/C:col10/1733344160509/Put/seqid=0 2024-12-04T20:29:22,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742248_1424 (size=12151) 2024-12-04T20:29:22,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/3d2556e5fe1b4e3baf6228dbd79d353f 2024-12-04T20:29:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e673f69444c54a09a8c28528906e8f00 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e673f69444c54a09a8c28528906e8f00 2024-12-04T20:29:22,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e673f69444c54a09a8c28528906e8f00, entries=200, sequenceid=229, filesize=38.8 K 2024-12-04T20:29:22,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/2f9973510fc947d4953f0fba88e7d58a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2f9973510fc947d4953f0fba88e7d58a 2024-12-04T20:29:22,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2f9973510fc947d4953f0fba88e7d58a, entries=150, sequenceid=229, filesize=11.9 K 2024-12-04T20:29:22,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/3d2556e5fe1b4e3baf6228dbd79d353f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/3d2556e5fe1b4e3baf6228dbd79d353f 2024-12-04T20:29:22,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/3d2556e5fe1b4e3baf6228dbd79d353f, entries=150, sequenceid=229, filesize=11.9 K 2024-12-04T20:29:22,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 00a3709f8e7a19ee64c7444672f69f55 in 909ms, sequenceid=229, compaction requested=true 2024-12-04T20:29:22,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:22,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:22,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:22,594 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:22,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:22,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:22,594 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:22,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:22,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:22,595 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:22,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:22,595 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/B is initiating minor compaction (all files) 2024-12-04T20:29:22,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/A is initiating minor compaction (all files) 2024-12-04T20:29:22,595 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/B in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:22,595 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/A in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:22,595 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/22e376cc5a5a4b26b1aff7f27712c5fe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/551009bafaf64ce7acd6de9a1e042d92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2f9973510fc947d4953f0fba88e7d58a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=36.0 K 2024-12-04T20:29:22,595 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/7aea34faf23f45d882729b527fedc901, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/295ba1a038a04d6f892044c12819e51c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e673f69444c54a09a8c28528906e8f00] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=100.0 K 2024-12-04T20:29:22,595 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:22,595 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/7aea34faf23f45d882729b527fedc901, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/295ba1a038a04d6f892044c12819e51c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e673f69444c54a09a8c28528906e8f00] 2024-12-04T20:29:22,595 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 22e376cc5a5a4b26b1aff7f27712c5fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733344159438 2024-12-04T20:29:22,596 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 551009bafaf64ce7acd6de9a1e042d92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733344159614 2024-12-04T20:29:22,596 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aea34faf23f45d882729b527fedc901, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733344159438 2024-12-04T20:29:22,596 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9973510fc947d4953f0fba88e7d58a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733344160509 2024-12-04T20:29:22,596 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 295ba1a038a04d6f892044c12819e51c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733344159614 2024-12-04T20:29:22,596 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e673f69444c54a09a8c28528906e8f00, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733344160509 2024-12-04T20:29:22,622 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:22,624 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:22,624 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/56865cca54e54c5f810df267400dc1c5 is 50, key is test_row_0/B:col10/1733344160509/Put/seqid=0 2024-12-04T20:29:22,629 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120470256b30cdae46f0807b843ed2e3e918_00a3709f8e7a19ee64c7444672f69f55 store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:22,631 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120470256b30cdae46f0807b843ed2e3e918_00a3709f8e7a19ee64c7444672f69f55, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:22,631 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120470256b30cdae46f0807b843ed2e3e918_00a3709f8e7a19ee64c7444672f69f55 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742249_1425 (size=12663) 2024-12-04T20:29:22,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742250_1426 (size=4469) 2024-12-04T20:29:22,644 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#A#compaction#357 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:22,645 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/56865cca54e54c5f810df267400dc1c5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/56865cca54e54c5f810df267400dc1c5 2024-12-04T20:29:22,645 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/df193383fc5f485081aba89a41cfc0b3 is 175, key is test_row_0/A:col10/1733344160509/Put/seqid=0 2024-12-04T20:29:22,649 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/B of 00a3709f8e7a19ee64c7444672f69f55 into 56865cca54e54c5f810df267400dc1c5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:22,649 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:22,649 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/B, priority=13, startTime=1733344162594; duration=0sec 2024-12-04T20:29:22,649 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:22,649 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:B 2024-12-04T20:29:22,650 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:22,652 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:22,652 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/C is initiating minor compaction (all files) 2024-12-04T20:29:22,652 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/C in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:22,652 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d929437b854f4932ac59895cb5b3b371, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6d4fd101fdc243c1a1da0ae2074a8347, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/3d2556e5fe1b4e3baf6228dbd79d353f] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=36.0 K 2024-12-04T20:29:22,653 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d929437b854f4932ac59895cb5b3b371, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733344159438 2024-12-04T20:29:22,653 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4fd101fdc243c1a1da0ae2074a8347, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733344159614 2024-12-04T20:29:22,653 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2556e5fe1b4e3baf6228dbd79d353f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733344160509 2024-12-04T20:29:22,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742251_1427 (size=31617) 2024-12-04T20:29:22,660 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#C#compaction#359 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:22,661 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e1668e92422e4c6e8590f828c0f1b0e7 is 50, key is test_row_0/C:col10/1733344160509/Put/seqid=0 2024-12-04T20:29:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742252_1428 (size=12663) 2024-12-04T20:29:22,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344161802, max timestamp=1733344162907Num uniques 1018; 2024-12-04T20:29:22,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344161802, max timestamp=1733344162907Num uniques 1018; 2024-12-04T20:29:22,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:29:22,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:29:22,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344161802, max timestamp=1733344162907Num uniques 1018; 2024-12-04T20:29:22,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:29:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:22,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-04T20:29:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:22,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043a4c4b2b49344a6caf886a7eb3346a42_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344162926/Put/seqid=0 2024-12-04T20:29:22,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742253_1429 (size=12454) 2024-12-04T20:29:22,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:22,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043a4c4b2b49344a6caf886a7eb3346a42_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043a4c4b2b49344a6caf886a7eb3346a42_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:22,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/35575950c69d45528f060d438d81b6f7, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:22,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/35575950c69d45528f060d438d81b6f7 is 175, key is test_row_0/A:col10/1733344162926/Put/seqid=0 2024-12-04T20:29:22,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742254_1430 (size=31255) 2024-12-04T20:29:23,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344163009, max timestamp=1733344163043Num uniques 859; 2024-12-04T20:29:23,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:23,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344163009, max timestamp=1733344163043Num uniques 859; 2024-12-04T20:29:23,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:23,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344163009, max timestamp=1733344163043Num uniques 859; 2024-12-04T20:29:23,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:29:23,062 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/df193383fc5f485081aba89a41cfc0b3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df193383fc5f485081aba89a41cfc0b3 2024-12-04T20:29:23,067 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/A of 00a3709f8e7a19ee64c7444672f69f55 into df193383fc5f485081aba89a41cfc0b3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:23,067 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:23,067 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/A, priority=13, startTime=1733344162594; duration=0sec 2024-12-04T20:29:23,067 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:23,067 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:A 2024-12-04T20:29:23,091 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/e1668e92422e4c6e8590f828c0f1b0e7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e1668e92422e4c6e8590f828c0f1b0e7 2024-12-04T20:29:23,101 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/C of 00a3709f8e7a19ee64c7444672f69f55 into e1668e92422e4c6e8590f828c0f1b0e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:23,101 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:23,101 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/C, priority=13, startTime=1733344162594; duration=0sec 2024-12-04T20:29:23,101 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:23,101 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:C 2024-12-04T20:29:23,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344223106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344223107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344223119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344223222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344223222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344223231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,369 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/35575950c69d45528f060d438d81b6f7 2024-12-04T20:29:23,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/48de2ecd075e47a69fc7217a3326808a is 50, key is test_row_0/B:col10/1733344162926/Put/seqid=0 2024-12-04T20:29:23,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742255_1431 (size=12301) 2024-12-04T20:29:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344223432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344223442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344223442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T20:29:23,534 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:23,534 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-04T20:29:23,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T20:29:23,537 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:23,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T20:29:23,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T20:29:23,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:23,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:23,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:23,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:23,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:23,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344223753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344223764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344223764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/48de2ecd075e47a69fc7217a3326808a 2024-12-04T20:29:23,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/ed973d2f615d45d18f6919753d31680e is 50, key is test_row_0/C:col10/1733344162926/Put/seqid=0 2024-12-04T20:29:23,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T20:29:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:23,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T20:29:23,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742256_1432 (size=12301) 2024-12-04T20:29:23,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/ed973d2f615d45d18f6919753d31680e 2024-12-04T20:29:23,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344223862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/35575950c69d45528f060d438d81b6f7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/35575950c69d45528f060d438d81b6f7 2024-12-04T20:29:23,874 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/35575950c69d45528f060d438d81b6f7, entries=150, sequenceid=272, filesize=30.5 K 2024-12-04T20:29:23,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/48de2ecd075e47a69fc7217a3326808a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/48de2ecd075e47a69fc7217a3326808a 2024-12-04T20:29:23,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344223872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:23,883 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:23,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/48de2ecd075e47a69fc7217a3326808a, entries=150, sequenceid=272, filesize=12.0 K 2024-12-04T20:29:23,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/ed973d2f615d45d18f6919753d31680e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/ed973d2f615d45d18f6919753d31680e 2024-12-04T20:29:23,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/ed973d2f615d45d18f6919753d31680e, entries=150, sequenceid=272, filesize=12.0 K 2024-12-04T20:29:23,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for 00a3709f8e7a19ee64c7444672f69f55 in 964ms, sequenceid=272, compaction requested=false 2024-12-04T20:29:23,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:23,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T20:29:23,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:23,999 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T20:29:24,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:24,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:24,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:24,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:24,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:24,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:24,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045e66c80a6123486eaaf35e25984d5566_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344163102/Put/seqid=0 2024-12-04T20:29:24,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742257_1433 (size=12454) 2024-12-04T20:29:24,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:24,076 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045e66c80a6123486eaaf35e25984d5566_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e66c80a6123486eaaf35e25984d5566_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:24,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e2cfbec18c1840c6a66f73204cd46c4e, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:24,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e2cfbec18c1840c6a66f73204cd46c4e is 175, key is test_row_0/A:col10/1733344163102/Put/seqid=0 2024-12-04T20:29:24,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742258_1434 (size=31255) 2024-12-04T20:29:24,091 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e2cfbec18c1840c6a66f73204cd46c4e 2024-12-04T20:29:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/f58f0be3eb50485dbbc8168f5e2f3789 is 50, key is test_row_0/B:col10/1733344163102/Put/seqid=0 2024-12-04T20:29:24,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742259_1435 (size=12301) 2024-12-04T20:29:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T20:29:24,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:24,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344224315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344224315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344224316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344224431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344224431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344224431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/f58f0be3eb50485dbbc8168f5e2f3789 2024-12-04T20:29:24,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c62bc5ab391944a08c6d0e52a4e379f0 is 50, key is test_row_0/C:col10/1733344163102/Put/seqid=0 2024-12-04T20:29:24,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742260_1436 (size=12301) 2024-12-04T20:29:24,519 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c62bc5ab391944a08c6d0e52a4e379f0 2024-12-04T20:29:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/e2cfbec18c1840c6a66f73204cd46c4e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e2cfbec18c1840c6a66f73204cd46c4e 2024-12-04T20:29:24,526 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e2cfbec18c1840c6a66f73204cd46c4e, entries=150, sequenceid=311, filesize=30.5 K 2024-12-04T20:29:24,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/f58f0be3eb50485dbbc8168f5e2f3789 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f58f0be3eb50485dbbc8168f5e2f3789 2024-12-04T20:29:24,532 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f58f0be3eb50485dbbc8168f5e2f3789, entries=150, sequenceid=311, filesize=12.0 K 2024-12-04T20:29:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c62bc5ab391944a08c6d0e52a4e379f0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c62bc5ab391944a08c6d0e52a4e379f0 2024-12-04T20:29:24,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c62bc5ab391944a08c6d0e52a4e379f0, entries=150, sequenceid=311, filesize=12.0 K 2024-12-04T20:29:24,541 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 00a3709f8e7a19ee64c7444672f69f55 in 542ms, sequenceid=311, compaction requested=true 2024-12-04T20:29:24,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:24,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:24,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-04T20:29:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-04T20:29:24,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-04T20:29:24,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-12-04T20:29:24,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0140 sec 2024-12-04T20:29:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:24,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:29:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:24,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120421a7dd184f514870b1657b939f2cc392_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344164284/Put/seqid=0 2024-12-04T20:29:24,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742261_1437 (size=14994) 2024-12-04T20:29:24,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:24,666 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120421a7dd184f514870b1657b939f2cc392_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120421a7dd184f514870b1657b939f2cc392_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:24,667 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/62d679a9b22c470793a10b819fb0039e, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:24,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/62d679a9b22c470793a10b819fb0039e is 175, key is test_row_0/A:col10/1733344164284/Put/seqid=0 2024-12-04T20:29:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T20:29:24,672 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:24,673 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-04T20:29:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T20:29:24,675 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:24,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:24,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:24,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742262_1438 (size=39949) 2024-12-04T20:29:24,686 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/62d679a9b22c470793a10b819fb0039e 2024-12-04T20:29:24,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/aeae4a4e0d7849cdb8e42f092c09e8f3 is 50, key is test_row_0/B:col10/1733344164284/Put/seqid=0 2024-12-04T20:29:24,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742263_1439 (size=12301) 2024-12-04T20:29:24,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344224718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344224719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344224719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T20:29:24,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T20:29:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:24,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344224832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344224833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344224833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:24,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T20:29:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:24,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:24,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:24,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:24,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T20:29:25,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344225042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344225045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344225052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/aeae4a4e0d7849cdb8e42f092c09e8f3 2024-12-04T20:29:25,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/d194997bf3a3440d8e6b27ce93cfcc57 is 50, key is test_row_0/C:col10/1733344164284/Put/seqid=0 2024-12-04T20:29:25,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742264_1440 (size=12301) 2024-12-04T20:29:25,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T20:29:25,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:25,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T20:29:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T20:29:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344225362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344225362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344225372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T20:29:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:25,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/d194997bf3a3440d8e6b27ce93cfcc57 2024-12-04T20:29:25,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/62d679a9b22c470793a10b819fb0039e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/62d679a9b22c470793a10b819fb0039e 2024-12-04T20:29:25,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/62d679a9b22c470793a10b819fb0039e, entries=200, sequenceid=323, filesize=39.0 K 2024-12-04T20:29:25,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/aeae4a4e0d7849cdb8e42f092c09e8f3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/aeae4a4e0d7849cdb8e42f092c09e8f3 2024-12-04T20:29:25,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/aeae4a4e0d7849cdb8e42f092c09e8f3, entries=150, sequenceid=323, filesize=12.0 K 2024-12-04T20:29:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/d194997bf3a3440d8e6b27ce93cfcc57 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d194997bf3a3440d8e6b27ce93cfcc57 2024-12-04T20:29:25,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d194997bf3a3440d8e6b27ce93cfcc57, entries=150, sequenceid=323, filesize=12.0 K 2024-12-04T20:29:25,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 00a3709f8e7a19ee64c7444672f69f55 in 902ms, sequenceid=323, compaction requested=true 2024-12-04T20:29:25,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:25,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:25,548 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:29:25,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:25,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:25,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:25,548 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:29:25,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:25,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:25,548 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134076 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:29:25,549 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/A is initiating minor compaction (all files) 2024-12-04T20:29:25,549 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/A in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,549 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:29:25,549 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/B is initiating minor compaction (all files) 2024-12-04T20:29:25,549 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df193383fc5f485081aba89a41cfc0b3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/35575950c69d45528f060d438d81b6f7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e2cfbec18c1840c6a66f73204cd46c4e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/62d679a9b22c470793a10b819fb0039e] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=130.9 K 2024-12-04T20:29:25,549 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/B in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,549 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,549 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/56865cca54e54c5f810df267400dc1c5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/48de2ecd075e47a69fc7217a3326808a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f58f0be3eb50485dbbc8168f5e2f3789, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/aeae4a4e0d7849cdb8e42f092c09e8f3] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=48.4 K 2024-12-04T20:29:25,549 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df193383fc5f485081aba89a41cfc0b3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/35575950c69d45528f060d438d81b6f7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e2cfbec18c1840c6a66f73204cd46c4e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/62d679a9b22c470793a10b819fb0039e] 2024-12-04T20:29:25,549 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 56865cca54e54c5f810df267400dc1c5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733344160509 2024-12-04T20:29:25,549 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting df193383fc5f485081aba89a41cfc0b3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733344160509 2024-12-04T20:29:25,550 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 48de2ecd075e47a69fc7217a3326808a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733344162907 2024-12-04T20:29:25,550 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35575950c69d45528f060d438d81b6f7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733344162907 2024-12-04T20:29:25,550 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f58f0be3eb50485dbbc8168f5e2f3789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733344163102 2024-12-04T20:29:25,550 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2cfbec18c1840c6a66f73204cd46c4e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733344163102 2024-12-04T20:29:25,550 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting aeae4a4e0d7849cdb8e42f092c09e8f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733344164284 2024-12-04T20:29:25,551 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62d679a9b22c470793a10b819fb0039e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733344164284 2024-12-04T20:29:25,561 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:25,569 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120430ec1ee6c0764a918d069d8b5f5409e9_00a3709f8e7a19ee64c7444672f69f55 store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:25,571 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:25,571 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120430ec1ee6c0764a918d069d8b5f5409e9_00a3709f8e7a19ee64c7444672f69f55, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:25,571 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/288fea366fd640e69edd0b1702808bbb is 50, key is test_row_0/B:col10/1733344164284/Put/seqid=0 2024-12-04T20:29:25,571 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120430ec1ee6c0764a918d069d8b5f5409e9_00a3709f8e7a19ee64c7444672f69f55 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:25,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742265_1441 (size=12949) 2024-12-04T20:29:25,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742266_1442 (size=4469) 2024-12-04T20:29:25,583 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#A#compaction#369 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:25,583 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/69de865d9fb2487e820f12d70cf5612a is 175, key is test_row_0/A:col10/1733344164284/Put/seqid=0 2024-12-04T20:29:25,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742267_1443 (size=31903) 2024-12-04T20:29:25,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T20:29:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,595 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T20:29:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:25,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:25,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:25,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:25,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:25,601 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/69de865d9fb2487e820f12d70cf5612a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/69de865d9fb2487e820f12d70cf5612a 2024-12-04T20:29:25,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045a20f9901bc44e37af1c0fb687eb4750_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344164702/Put/seqid=0 2024-12-04T20:29:25,608 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/A of 00a3709f8e7a19ee64c7444672f69f55 into 69de865d9fb2487e820f12d70cf5612a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:25,608 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:25,608 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/A, priority=12, startTime=1733344165548; duration=0sec 2024-12-04T20:29:25,608 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:25,608 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:A 2024-12-04T20:29:25,608 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:29:25,609 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:29:25,609 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/C is initiating minor compaction (all files) 2024-12-04T20:29:25,609 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/C in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:25,609 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e1668e92422e4c6e8590f828c0f1b0e7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/ed973d2f615d45d18f6919753d31680e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c62bc5ab391944a08c6d0e52a4e379f0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d194997bf3a3440d8e6b27ce93cfcc57] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=48.4 K 2024-12-04T20:29:25,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742268_1444 (size=12454) 2024-12-04T20:29:25,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1668e92422e4c6e8590f828c0f1b0e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733344160509 2024-12-04T20:29:25,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed973d2f615d45d18f6919753d31680e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733344162907 2024-12-04T20:29:25,610 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c62bc5ab391944a08c6d0e52a4e379f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733344163102 2024-12-04T20:29:25,611 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d194997bf3a3440d8e6b27ce93cfcc57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733344164284 2024-12-04T20:29:25,617 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:25,618 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/54ea578319c34a379bdfbe79f097d9c7 is 50, key is test_row_0/C:col10/1733344164284/Put/seqid=0 2024-12-04T20:29:25,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742269_1445 (size=12949) 2024-12-04T20:29:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T20:29:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:25,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. as already flushing 2024-12-04T20:29:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344225906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344225906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344225907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:25,984 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/288fea366fd640e69edd0b1702808bbb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/288fea366fd640e69edd0b1702808bbb 2024-12-04T20:29:25,988 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/B of 00a3709f8e7a19ee64c7444672f69f55 into 288fea366fd640e69edd0b1702808bbb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:25,988 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:25,988 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/B, priority=12, startTime=1733344165548; duration=0sec 2024-12-04T20:29:25,988 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:25,988 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:B 2024-12-04T20:29:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:26,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045a20f9901bc44e37af1c0fb687eb4750_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045a20f9901bc44e37af1c0fb687eb4750_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:26,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/23e19796d02242f68301c1414b6b2e8c, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:26,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/23e19796d02242f68301c1414b6b2e8c is 175, key is test_row_0/A:col10/1733344164702/Put/seqid=0 2024-12-04T20:29:26,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742270_1446 (size=31255) 2024-12-04T20:29:26,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344226021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344226021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344226022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,028 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/54ea578319c34a379bdfbe79f097d9c7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/54ea578319c34a379bdfbe79f097d9c7 2024-12-04T20:29:26,032 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/C of 00a3709f8e7a19ee64c7444672f69f55 into 54ea578319c34a379bdfbe79f097d9c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:26,032 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:26,032 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/C, priority=12, startTime=1733344165548; duration=0sec 2024-12-04T20:29:26,032 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:26,032 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:C 2024-12-04T20:29:26,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344226231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344226231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344226231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/23e19796d02242f68301c1414b6b2e8c 2024-12-04T20:29:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/20382001208d47ca99deb64b5c2424e6 is 50, key is test_row_0/B:col10/1733344164702/Put/seqid=0 2024-12-04T20:29:26,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742271_1447 (size=12301) 2024-12-04T20:29:26,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344226541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344226542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344226551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T20:29:26,835 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/20382001208d47ca99deb64b5c2424e6 2024-12-04T20:29:26,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/09c73bbfe09b4d0ea8e1c41265dcd55b is 50, key is test_row_0/C:col10/1733344164702/Put/seqid=0 2024-12-04T20:29:26,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742272_1448 (size=12301) 2024-12-04T20:29:26,848 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/09c73bbfe09b4d0ea8e1c41265dcd55b 2024-12-04T20:29:26,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/23e19796d02242f68301c1414b6b2e8c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/23e19796d02242f68301c1414b6b2e8c 2024-12-04T20:29:26,855 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/23e19796d02242f68301c1414b6b2e8c, entries=150, sequenceid=343, filesize=30.5 K 2024-12-04T20:29:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/20382001208d47ca99deb64b5c2424e6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/20382001208d47ca99deb64b5c2424e6 2024-12-04T20:29:26,860 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/20382001208d47ca99deb64b5c2424e6, entries=150, sequenceid=343, filesize=12.0 K 2024-12-04T20:29:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/09c73bbfe09b4d0ea8e1c41265dcd55b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/09c73bbfe09b4d0ea8e1c41265dcd55b 2024-12-04T20:29:26,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/09c73bbfe09b4d0ea8e1c41265dcd55b, entries=150, sequenceid=343, filesize=12.0 K 2024-12-04T20:29:26,874 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 00a3709f8e7a19ee64c7444672f69f55 in 1278ms, sequenceid=343, compaction requested=false 2024-12-04T20:29:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-04T20:29:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-04T20:29:26,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-04T20:29:26,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-12-04T20:29:26,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2060 sec 2024-12-04T20:29:27,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:29:27,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:27,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:27,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:27,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:27,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:27,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:27,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ff8d1df65a7b4fd8ba387bb87ac4f593_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344165875/Put/seqid=0 2024-12-04T20:29:27,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742273_1449 (size=14994) 2024-12-04T20:29:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344227127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344227131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344227135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344227242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344227242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344227251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344227462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344227462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344227462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,519 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:27,522 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ff8d1df65a7b4fd8ba387bb87ac4f593_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ff8d1df65a7b4fd8ba387bb87ac4f593_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:27,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/be3e6ec1f7e74797adc79c3a06adde5c, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:27,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/be3e6ec1f7e74797adc79c3a06adde5c is 175, key is test_row_0/A:col10/1733344165875/Put/seqid=0 2024-12-04T20:29:27,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742274_1450 (size=39949) 2024-12-04T20:29:27,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344227781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344227781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344227781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36282 deadline: 1733344227892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,898 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1733344227901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:27,910 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in 00a3709f8e7a19ee64c7444672f69f55 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:27,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/be3e6ec1f7e74797adc79c3a06adde5c 2024-12-04T20:29:27,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/3f0b4e07002b4ae58bd9649aaed27c43 is 50, key is test_row_0/B:col10/1733344165875/Put/seqid=0 2024-12-04T20:29:27,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742275_1451 (size=12301) 2024-12-04T20:29:28,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1733344228292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:28,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36192 deadline: 1733344228292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1733344228301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:29:28,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/3f0b4e07002b4ae58bd9649aaed27c43 2024-12-04T20:29:28,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c390c7a82db64e92b603304566c7d2d9 is 50, key is test_row_0/C:col10/1733344165875/Put/seqid=0 2024-12-04T20:29:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742276_1452 (size=12301) 2024-12-04T20:29:28,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c390c7a82db64e92b603304566c7d2d9 2024-12-04T20:29:28,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/be3e6ec1f7e74797adc79c3a06adde5c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/be3e6ec1f7e74797adc79c3a06adde5c 2024-12-04T20:29:28,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/be3e6ec1f7e74797adc79c3a06adde5c, entries=200, sequenceid=359, filesize=39.0 K 2024-12-04T20:29:28,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/3f0b4e07002b4ae58bd9649aaed27c43 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/3f0b4e07002b4ae58bd9649aaed27c43 2024-12-04T20:29:28,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/3f0b4e07002b4ae58bd9649aaed27c43, entries=150, sequenceid=359, filesize=12.0 K 2024-12-04T20:29:28,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/c390c7a82db64e92b603304566c7d2d9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c390c7a82db64e92b603304566c7d2d9 2024-12-04T20:29:28,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c390c7a82db64e92b603304566c7d2d9, entries=150, sequenceid=359, filesize=12.0 K 2024-12-04T20:29:28,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 00a3709f8e7a19ee64c7444672f69f55 in 1708ms, sequenceid=359, compaction requested=true 2024-12-04T20:29:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:28,775 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:28,776 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a3709f8e7a19ee64c7444672f69f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:28,776 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:28,776 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:28,776 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/A is initiating minor compaction (all files) 2024-12-04T20:29:28,776 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/B is initiating minor compaction (all files) 2024-12-04T20:29:28,777 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/B in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:28,777 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/A in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:28,777 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/288fea366fd640e69edd0b1702808bbb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/20382001208d47ca99deb64b5c2424e6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/3f0b4e07002b4ae58bd9649aaed27c43] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=36.7 K 2024-12-04T20:29:28,777 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/69de865d9fb2487e820f12d70cf5612a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/23e19796d02242f68301c1414b6b2e8c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/be3e6ec1f7e74797adc79c3a06adde5c] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=100.7 K 2024-12-04T20:29:28,777 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:28,777 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/69de865d9fb2487e820f12d70cf5612a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/23e19796d02242f68301c1414b6b2e8c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/be3e6ec1f7e74797adc79c3a06adde5c] 2024-12-04T20:29:28,777 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 288fea366fd640e69edd0b1702808bbb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733344164284 2024-12-04T20:29:28,777 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69de865d9fb2487e820f12d70cf5612a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733344164284 2024-12-04T20:29:28,777 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 20382001208d47ca99deb64b5c2424e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733344164702 2024-12-04T20:29:28,777 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e19796d02242f68301c1414b6b2e8c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733344164702 2024-12-04T20:29:28,777 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0b4e07002b4ae58bd9649aaed27c43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733344165875 2024-12-04T20:29:28,777 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting be3e6ec1f7e74797adc79c3a06adde5c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733344165875 2024-12-04T20:29:28,779 INFO [Thread-1662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1662. 2024-12-04T20:29:28,779 DEBUG [Thread-1662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:28,780 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,780 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:28,782 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-12-04T20:29:28,782 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:28,782 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,782 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:28,783 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:28,783 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0e1f7bc29b5546338ca3c6367f8c3ecf is 50, key is test_row_0/B:col10/1733344165875/Put/seqid=0 2024-12-04T20:29:28,785 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-12-04T20:29:28,785 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:28,785 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,785 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:28,786 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:28,787 INFO [Thread-1664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1664. 2024-12-04T20:29:28,787 DEBUG [Thread-1664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:28,787 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,787 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:28,790 INFO [Thread-1660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1660. 2024-12-04T20:29:28,790 DEBUG [Thread-1660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:28,790 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,790 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:28,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:28,790 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412040cbb2b6516b8498a8e2a1f5dc3d6cd4e_00a3709f8e7a19ee64c7444672f69f55 store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:28,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742277_1453 (size=13051) 2024-12-04T20:29:28,791 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412040cbb2b6516b8498a8e2a1f5dc3d6cd4e_00a3709f8e7a19ee64c7444672f69f55, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:28,791 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040cbb2b6516b8498a8e2a1f5dc3d6cd4e_00a3709f8e7a19ee64c7444672f69f55 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:28,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742278_1454 (size=4469) 2024-12-04T20:29:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T20:29:28,842 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:29,196 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#A#compaction#379 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:29,197 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/f2ae87be627345ff9c7a9cdb81f054a3 is 175, key is test_row_0/A:col10/1733344165875/Put/seqid=0 2024-12-04T20:29:29,199 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/0e1f7bc29b5546338ca3c6367f8c3ecf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e1f7bc29b5546338ca3c6367f8c3ecf 2024-12-04T20:29:29,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742279_1455 (size=32005) 2024-12-04T20:29:29,206 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/B of 00a3709f8e7a19ee64c7444672f69f55 into 0e1f7bc29b5546338ca3c6367f8c3ecf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:29,206 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:29,206 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/B, priority=13, startTime=1733344168775; duration=0sec 2024-12-04T20:29:29,206 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:29,206 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:B 2024-12-04T20:29:29,206 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:29,207 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:29,207 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 00a3709f8e7a19ee64c7444672f69f55/C is initiating minor compaction (all files) 2024-12-04T20:29:29,207 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a3709f8e7a19ee64c7444672f69f55/C in TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:29,208 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/54ea578319c34a379bdfbe79f097d9c7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/09c73bbfe09b4d0ea8e1c41265dcd55b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c390c7a82db64e92b603304566c7d2d9] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp, totalSize=36.7 K 2024-12-04T20:29:29,208 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ea578319c34a379bdfbe79f097d9c7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733344164284 2024-12-04T20:29:29,208 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c73bbfe09b4d0ea8e1c41265dcd55b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733344164702 2024-12-04T20:29:29,209 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c390c7a82db64e92b603304566c7d2d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733344165875 2024-12-04T20:29:29,216 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a3709f8e7a19ee64c7444672f69f55#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:29,216 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/dc13242d888d4e67893dca9b490fec5d is 50, key is test_row_0/C:col10/1733344165875/Put/seqid=0 2024-12-04T20:29:29,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742280_1456 (size=13051) 2024-12-04T20:29:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:29,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T20:29:29,316 INFO [Thread-1645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1645. 2024-12-04T20:29:29,316 INFO [Thread-1653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1653. 2024-12-04T20:29:29,316 DEBUG [Thread-1645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:29,316 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:29,316 DEBUG [Thread-1653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:29,316 DEBUG [Thread-1653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:29,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:29,317 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:29,317 DEBUG [Thread-1653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:29,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:29,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:29,323 INFO [Thread-1649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1649. 2024-12-04T20:29:29,323 DEBUG [Thread-1649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:29,323 DEBUG [Thread-1649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:29,323 DEBUG [Thread-1649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:29,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:29,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120453d021e25bcb495eb492914502917c54_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_0/A:col10/1733344169313/Put/seqid=0 2024-12-04T20:29:29,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742281_1457 (size=12454) 2024-12-04T20:29:29,614 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/f2ae87be627345ff9c7a9cdb81f054a3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2ae87be627345ff9c7a9cdb81f054a3 2024-12-04T20:29:29,619 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/A of 00a3709f8e7a19ee64c7444672f69f55 into f2ae87be627345ff9c7a9cdb81f054a3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:29,619 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:29,619 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/A, priority=13, startTime=1733344168775; duration=0sec 2024-12-04T20:29:29,619 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:29,619 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:A 2024-12-04T20:29:29,625 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/dc13242d888d4e67893dca9b490fec5d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/dc13242d888d4e67893dca9b490fec5d 2024-12-04T20:29:29,630 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a3709f8e7a19ee64c7444672f69f55/C of 00a3709f8e7a19ee64c7444672f69f55 into dc13242d888d4e67893dca9b490fec5d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:29,630 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:29,630 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., storeName=00a3709f8e7a19ee64c7444672f69f55/C, priority=13, startTime=1733344168776; duration=0sec 2024-12-04T20:29:29,630 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:29,630 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a3709f8e7a19ee64c7444672f69f55:C 2024-12-04T20:29:29,729 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:29,737 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120453d021e25bcb495eb492914502917c54_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120453d021e25bcb495eb492914502917c54_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:29,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/819a5b3e51d240229f0cd53736857884, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:29,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/819a5b3e51d240229f0cd53736857884 is 175, key is test_row_0/A:col10/1733344169313/Put/seqid=0 2024-12-04T20:29:29,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742282_1458 (size=31255) 2024-12-04T20:29:30,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/819a5b3e51d240229f0cd53736857884 2024-12-04T20:29:30,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/dd36b52cdd5d49228bf38ae6cd2534c4 is 50, key is test_row_0/B:col10/1733344169313/Put/seqid=0 2024-12-04T20:29:30,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742283_1459 (size=12301) 2024-12-04T20:29:30,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/dd36b52cdd5d49228bf38ae6cd2534c4 2024-12-04T20:29:30,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/736f3569894a4cd1a6ab684f7591a473 is 50, key is test_row_0/C:col10/1733344169313/Put/seqid=0 2024-12-04T20:29:30,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742284_1460 (size=12301) 2024-12-04T20:29:30,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/736f3569894a4cd1a6ab684f7591a473 2024-12-04T20:29:30,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/819a5b3e51d240229f0cd53736857884 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/819a5b3e51d240229f0cd53736857884 2024-12-04T20:29:30,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/819a5b3e51d240229f0cd53736857884, entries=150, sequenceid=381, filesize=30.5 K 2024-12-04T20:29:30,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/dd36b52cdd5d49228bf38ae6cd2534c4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/dd36b52cdd5d49228bf38ae6cd2534c4 2024-12-04T20:29:30,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/dd36b52cdd5d49228bf38ae6cd2534c4, entries=150, sequenceid=381, filesize=12.0 K 2024-12-04T20:29:31,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/736f3569894a4cd1a6ab684f7591a473 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/736f3569894a4cd1a6ab684f7591a473 2024-12-04T20:29:31,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/736f3569894a4cd1a6ab684f7591a473, entries=150, sequenceid=381, filesize=12.0 K 2024-12-04T20:29:31,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=6.71 KB/6870 for 00a3709f8e7a19ee64c7444672f69f55 in 1688ms, sequenceid=381, compaction requested=false 2024-12-04T20:29:31,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a3709f8e7a19ee64c7444672f69f55: 2024-12-04T20:29:32,787 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:29:36,230 WARN [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-04T20:29:36,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/2150321162594b0aa28055f622f2bd19, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/9539ee62d743480f9ca32022e43d7642, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/c93839cf9bfb497592bca1a3ff997fcf, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3015fa82eef34a1f984a4a46bae7822b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/81adb66445eb45a79eb891d433b8bc6e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df0c4504fa534dfc8497993de5539727, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3271537e01b044ac89c37daef20e0786, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2795b6c9aae40ada746f16b502ee2dc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/a6f0f6a63b18497db41ebef32f37bcce, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b8e883d902804e369f42ceacb50cd178, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/642e0ea8954d4e89902465a3a2c25c59, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/26a74ad7b0564b46ae18aeed2f988b8d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/7aea34faf23f45d882729b527fedc901, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/295ba1a038a04d6f892044c12819e51c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e673f69444c54a09a8c28528906e8f00, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df193383fc5f485081aba89a41cfc0b3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/35575950c69d45528f060d438d81b6f7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e2cfbec18c1840c6a66f73204cd46c4e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/62d679a9b22c470793a10b819fb0039e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/69de865d9fb2487e820f12d70cf5612a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/23e19796d02242f68301c1414b6b2e8c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/be3e6ec1f7e74797adc79c3a06adde5c] to archive 2024-12-04T20:29:36,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:29:36,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/2150321162594b0aa28055f622f2bd19 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/2150321162594b0aa28055f622f2bd19 2024-12-04T20:29:36,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/9539ee62d743480f9ca32022e43d7642 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/9539ee62d743480f9ca32022e43d7642 2024-12-04T20:29:36,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/c93839cf9bfb497592bca1a3ff997fcf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/c93839cf9bfb497592bca1a3ff997fcf 2024-12-04T20:29:36,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3015fa82eef34a1f984a4a46bae7822b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3015fa82eef34a1f984a4a46bae7822b 2024-12-04T20:29:36,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/81adb66445eb45a79eb891d433b8bc6e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/81adb66445eb45a79eb891d433b8bc6e 2024-12-04T20:29:36,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df0c4504fa534dfc8497993de5539727 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df0c4504fa534dfc8497993de5539727 2024-12-04T20:29:36,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3271537e01b044ac89c37daef20e0786 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/3271537e01b044ac89c37daef20e0786 2024-12-04T20:29:36,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2795b6c9aae40ada746f16b502ee2dc to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2795b6c9aae40ada746f16b502ee2dc 2024-12-04T20:29:36,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/a6f0f6a63b18497db41ebef32f37bcce to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/a6f0f6a63b18497db41ebef32f37bcce 2024-12-04T20:29:36,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b8e883d902804e369f42ceacb50cd178 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b8e883d902804e369f42ceacb50cd178 2024-12-04T20:29:36,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/642e0ea8954d4e89902465a3a2c25c59 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/642e0ea8954d4e89902465a3a2c25c59 2024-12-04T20:29:36,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/26a74ad7b0564b46ae18aeed2f988b8d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/26a74ad7b0564b46ae18aeed2f988b8d 2024-12-04T20:29:36,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/7aea34faf23f45d882729b527fedc901 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/7aea34faf23f45d882729b527fedc901 2024-12-04T20:29:36,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/295ba1a038a04d6f892044c12819e51c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/295ba1a038a04d6f892044c12819e51c 2024-12-04T20:29:36,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e673f69444c54a09a8c28528906e8f00 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e673f69444c54a09a8c28528906e8f00 2024-12-04T20:29:36,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df193383fc5f485081aba89a41cfc0b3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/df193383fc5f485081aba89a41cfc0b3 2024-12-04T20:29:36,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/35575950c69d45528f060d438d81b6f7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/35575950c69d45528f060d438d81b6f7 2024-12-04T20:29:36,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e2cfbec18c1840c6a66f73204cd46c4e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/e2cfbec18c1840c6a66f73204cd46c4e 2024-12-04T20:29:36,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/62d679a9b22c470793a10b819fb0039e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/62d679a9b22c470793a10b819fb0039e 2024-12-04T20:29:36,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/69de865d9fb2487e820f12d70cf5612a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/69de865d9fb2487e820f12d70cf5612a 2024-12-04T20:29:36,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/23e19796d02242f68301c1414b6b2e8c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/23e19796d02242f68301c1414b6b2e8c 2024-12-04T20:29:36,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/be3e6ec1f7e74797adc79c3a06adde5c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/be3e6ec1f7e74797adc79c3a06adde5c 2024-12-04T20:29:36,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a83393351b2e4e64a6a08a4c644678e1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0dd05eae0fcd4031bff57d0f746d7fe5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/8bff761991c94767903b3c49ec7e7ed7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/62c2d6f733004888bc58550667aa80fd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e6a3c3642944d998d6b13b5f0134a99, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/15e64de3c8714664871378f558cfef54, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f0e79383acf348e682b476781cf7acad, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/fc8d69cfec524ebdbffa9f0300e239bd, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/21417e2ac3634efa8c1cb9e192b15741, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2e44cb7122b24eb2a2dce6e29fdbd462, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/da51d8bc80284bbb90840f666a4a9e14, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/22e376cc5a5a4b26b1aff7f27712c5fe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a42c473793c44ef8adeaba602517cc2d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/551009bafaf64ce7acd6de9a1e042d92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/56865cca54e54c5f810df267400dc1c5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2f9973510fc947d4953f0fba88e7d58a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/48de2ecd075e47a69fc7217a3326808a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f58f0be3eb50485dbbc8168f5e2f3789, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/288fea366fd640e69edd0b1702808bbb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/aeae4a4e0d7849cdb8e42f092c09e8f3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/20382001208d47ca99deb64b5c2424e6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/3f0b4e07002b4ae58bd9649aaed27c43] to archive 2024-12-04T20:29:36,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:29:36,258 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a83393351b2e4e64a6a08a4c644678e1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a83393351b2e4e64a6a08a4c644678e1 2024-12-04T20:29:36,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0dd05eae0fcd4031bff57d0f746d7fe5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0dd05eae0fcd4031bff57d0f746d7fe5 2024-12-04T20:29:36,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/8bff761991c94767903b3c49ec7e7ed7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/8bff761991c94767903b3c49ec7e7ed7 2024-12-04T20:29:36,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/62c2d6f733004888bc58550667aa80fd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/62c2d6f733004888bc58550667aa80fd 2024-12-04T20:29:36,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e6a3c3642944d998d6b13b5f0134a99 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e6a3c3642944d998d6b13b5f0134a99 2024-12-04T20:29:36,262 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/15e64de3c8714664871378f558cfef54 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/15e64de3c8714664871378f558cfef54 2024-12-04T20:29:36,262 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f0e79383acf348e682b476781cf7acad to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f0e79383acf348e682b476781cf7acad 2024-12-04T20:29:36,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/fc8d69cfec524ebdbffa9f0300e239bd to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/fc8d69cfec524ebdbffa9f0300e239bd 2024-12-04T20:29:36,264 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/21417e2ac3634efa8c1cb9e192b15741 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/21417e2ac3634efa8c1cb9e192b15741 2024-12-04T20:29:36,264 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2e44cb7122b24eb2a2dce6e29fdbd462 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2e44cb7122b24eb2a2dce6e29fdbd462 2024-12-04T20:29:36,265 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/da51d8bc80284bbb90840f666a4a9e14 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/da51d8bc80284bbb90840f666a4a9e14 2024-12-04T20:29:36,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/22e376cc5a5a4b26b1aff7f27712c5fe to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/22e376cc5a5a4b26b1aff7f27712c5fe 2024-12-04T20:29:36,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a42c473793c44ef8adeaba602517cc2d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/a42c473793c44ef8adeaba602517cc2d 2024-12-04T20:29:36,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/551009bafaf64ce7acd6de9a1e042d92 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/551009bafaf64ce7acd6de9a1e042d92 2024-12-04T20:29:36,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/56865cca54e54c5f810df267400dc1c5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/56865cca54e54c5f810df267400dc1c5 2024-12-04T20:29:36,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2f9973510fc947d4953f0fba88e7d58a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/2f9973510fc947d4953f0fba88e7d58a 2024-12-04T20:29:36,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/48de2ecd075e47a69fc7217a3326808a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/48de2ecd075e47a69fc7217a3326808a 2024-12-04T20:29:36,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f58f0be3eb50485dbbc8168f5e2f3789 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/f58f0be3eb50485dbbc8168f5e2f3789 2024-12-04T20:29:36,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/288fea366fd640e69edd0b1702808bbb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/288fea366fd640e69edd0b1702808bbb 2024-12-04T20:29:36,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/aeae4a4e0d7849cdb8e42f092c09e8f3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/aeae4a4e0d7849cdb8e42f092c09e8f3 2024-12-04T20:29:36,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/20382001208d47ca99deb64b5c2424e6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/20382001208d47ca99deb64b5c2424e6 2024-12-04T20:29:36,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/3f0b4e07002b4ae58bd9649aaed27c43 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/3f0b4e07002b4ae58bd9649aaed27c43 2024-12-04T20:29:36,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/8b31e869ba3d41baa2f550ff4a378467, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c16be74ca94549a29f5b96ba2da54888, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/30b0ef8ddd664441980704785ae0c081, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e076bbfb64144146ae3bf72dae125117, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6ed3a4c2ca7b47adaece658b7625e7db, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/b4b19aec4e43484fad79b69ece8c6081, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e43e6d02520c42e2bc21f3b65d86bc8f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/41e952596eab436bbd971ee1fa3d0081, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/9dd2ad2e2db843c8870b5cff29b11999, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/68f78469a2be4d3881893d3153c6ab83, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/0a8e5ebc3cdc4d048e2e2d1898050c32, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d929437b854f4932ac59895cb5b3b371, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/204718b76af346d7971d1e0bfaf4f03a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6d4fd101fdc243c1a1da0ae2074a8347, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e1668e92422e4c6e8590f828c0f1b0e7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/3d2556e5fe1b4e3baf6228dbd79d353f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/ed973d2f615d45d18f6919753d31680e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c62bc5ab391944a08c6d0e52a4e379f0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/54ea578319c34a379bdfbe79f097d9c7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d194997bf3a3440d8e6b27ce93cfcc57, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/09c73bbfe09b4d0ea8e1c41265dcd55b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c390c7a82db64e92b603304566c7d2d9] to archive 2024-12-04T20:29:36,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:29:36,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/8b31e869ba3d41baa2f550ff4a378467 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/8b31e869ba3d41baa2f550ff4a378467 2024-12-04T20:29:36,277 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c16be74ca94549a29f5b96ba2da54888 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c16be74ca94549a29f5b96ba2da54888 2024-12-04T20:29:36,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/30b0ef8ddd664441980704785ae0c081 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/30b0ef8ddd664441980704785ae0c081 2024-12-04T20:29:36,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e076bbfb64144146ae3bf72dae125117 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e076bbfb64144146ae3bf72dae125117 2024-12-04T20:29:36,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6ed3a4c2ca7b47adaece658b7625e7db to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6ed3a4c2ca7b47adaece658b7625e7db 2024-12-04T20:29:36,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/b4b19aec4e43484fad79b69ece8c6081 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/b4b19aec4e43484fad79b69ece8c6081 2024-12-04T20:29:36,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e43e6d02520c42e2bc21f3b65d86bc8f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e43e6d02520c42e2bc21f3b65d86bc8f 2024-12-04T20:29:36,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/41e952596eab436bbd971ee1fa3d0081 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/41e952596eab436bbd971ee1fa3d0081 2024-12-04T20:29:36,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/9dd2ad2e2db843c8870b5cff29b11999 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/9dd2ad2e2db843c8870b5cff29b11999 2024-12-04T20:29:36,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/68f78469a2be4d3881893d3153c6ab83 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/68f78469a2be4d3881893d3153c6ab83 2024-12-04T20:29:36,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/0a8e5ebc3cdc4d048e2e2d1898050c32 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/0a8e5ebc3cdc4d048e2e2d1898050c32 2024-12-04T20:29:36,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d929437b854f4932ac59895cb5b3b371 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d929437b854f4932ac59895cb5b3b371 2024-12-04T20:29:36,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/204718b76af346d7971d1e0bfaf4f03a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/204718b76af346d7971d1e0bfaf4f03a 2024-12-04T20:29:36,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6d4fd101fdc243c1a1da0ae2074a8347 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/6d4fd101fdc243c1a1da0ae2074a8347 2024-12-04T20:29:36,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e1668e92422e4c6e8590f828c0f1b0e7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/e1668e92422e4c6e8590f828c0f1b0e7 2024-12-04T20:29:36,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/3d2556e5fe1b4e3baf6228dbd79d353f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/3d2556e5fe1b4e3baf6228dbd79d353f 2024-12-04T20:29:36,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/ed973d2f615d45d18f6919753d31680e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/ed973d2f615d45d18f6919753d31680e 2024-12-04T20:29:36,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c62bc5ab391944a08c6d0e52a4e379f0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c62bc5ab391944a08c6d0e52a4e379f0 2024-12-04T20:29:36,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/54ea578319c34a379bdfbe79f097d9c7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/54ea578319c34a379bdfbe79f097d9c7 2024-12-04T20:29:36,289 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d194997bf3a3440d8e6b27ce93cfcc57 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/d194997bf3a3440d8e6b27ce93cfcc57 2024-12-04T20:29:36,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/09c73bbfe09b4d0ea8e1c41265dcd55b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/09c73bbfe09b4d0ea8e1c41265dcd55b 2024-12-04T20:29:36,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c390c7a82db64e92b603304566c7d2d9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/c390c7a82db64e92b603304566c7d2d9 2024-12-04T20:29:37,924 INFO [Thread-1651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1651. 2024-12-04T20:29:37,924 DEBUG [Thread-1651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:37,924 DEBUG [Thread-1651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:37,924 DEBUG [Thread-1651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:37,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:37,954 INFO [Thread-1647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1647. 2024-12-04T20:29:37,954 DEBUG [Thread-1647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:29:37,954 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:37,954 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:37,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1410 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4230 rows 2024-12-04T20:29:37,954 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1411 2024-12-04T20:29:37,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4233 rows 2024-12-04T20:29:37,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1410 2024-12-04T20:29:37,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4230 rows 2024-12-04T20:29:37,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1413 2024-12-04T20:29:37,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4239 rows 2024-12-04T20:29:37,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1405 2024-12-04T20:29:37,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4214 rows 2024-12-04T20:29:37,955 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:29:37,955 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:29:37,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:37,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:37,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:29:37,956 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T20:29:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T20:29:37,963 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344177963"}]},"ts":"1733344177963"} 2024-12-04T20:29:37,965 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T20:29:37,965 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T20:29:37,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:29:37,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, UNASSIGN}] 2024-12-04T20:29:37,968 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, UNASSIGN 2024-12-04T20:29:37,969 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:37,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, UNASSIGN because future has completed 2024-12-04T20:29:37,970 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:29:37,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:29:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T20:29:38,124 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:38,124 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:29:38,124 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 00a3709f8e7a19ee64c7444672f69f55, disabling compactions & flushes 2024-12-04T20:29:38,124 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:38,124 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:38,125 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. after waiting 0 ms 2024-12-04T20:29:38,125 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:38,125 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 00a3709f8e7a19ee64c7444672f69f55 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-04T20:29:38,126 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=A 2024-12-04T20:29:38,126 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:38,126 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=B 2024-12-04T20:29:38,126 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:38,126 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a3709f8e7a19ee64c7444672f69f55, store=C 2024-12-04T20:29:38,126 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:38,135 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465e95f4f54724f86a9598c5ddce17419_00a3709f8e7a19ee64c7444672f69f55 is 50, key is test_row_1/A:col10/1733344169322/Put/seqid=0 2024-12-04T20:29:38,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742285_1461 (size=9914) 2024-12-04T20:29:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T20:29:38,540 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:38,550 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465e95f4f54724f86a9598c5ddce17419_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465e95f4f54724f86a9598c5ddce17419_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:38,551 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b3394a10daab4d0ba93de20e1396552d, store: [table=TestAcidGuarantees family=A region=00a3709f8e7a19ee64c7444672f69f55] 2024-12-04T20:29:38,552 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b3394a10daab4d0ba93de20e1396552d is 175, key is test_row_1/A:col10/1733344169322/Put/seqid=0 2024-12-04T20:29:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742286_1462 (size=22561) 2024-12-04T20:29:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T20:29:38,958 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b3394a10daab4d0ba93de20e1396552d 2024-12-04T20:29:38,968 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/623a278643b04ac299866b7e511ccb97 is 50, key is test_row_1/B:col10/1733344169322/Put/seqid=0 2024-12-04T20:29:38,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742287_1463 (size=9857) 2024-12-04T20:29:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T20:29:39,375 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/623a278643b04ac299866b7e511ccb97 2024-12-04T20:29:39,389 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/5c5cfef0eb6a4a1bbcd3025f7f725959 is 50, key is test_row_1/C:col10/1733344169322/Put/seqid=0 2024-12-04T20:29:39,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742288_1464 (size=9857) 2024-12-04T20:29:39,794 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/5c5cfef0eb6a4a1bbcd3025f7f725959 2024-12-04T20:29:39,804 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/A/b3394a10daab4d0ba93de20e1396552d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b3394a10daab4d0ba93de20e1396552d 2024-12-04T20:29:39,810 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b3394a10daab4d0ba93de20e1396552d, entries=100, sequenceid=389, filesize=22.0 K 2024-12-04T20:29:39,811 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/B/623a278643b04ac299866b7e511ccb97 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/623a278643b04ac299866b7e511ccb97 2024-12-04T20:29:39,815 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/623a278643b04ac299866b7e511ccb97, entries=100, sequenceid=389, filesize=9.6 K 2024-12-04T20:29:39,816 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/.tmp/C/5c5cfef0eb6a4a1bbcd3025f7f725959 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/5c5cfef0eb6a4a1bbcd3025f7f725959 2024-12-04T20:29:39,820 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/5c5cfef0eb6a4a1bbcd3025f7f725959, entries=100, sequenceid=389, filesize=9.6 K 2024-12-04T20:29:39,821 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00a3709f8e7a19ee64c7444672f69f55 in 1696ms, sequenceid=389, compaction requested=true 2024-12-04T20:29:39,827 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/recovered.edits/392.seqid, newMaxSeqId=392, maxSeqId=4 2024-12-04T20:29:39,828 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. 2024-12-04T20:29:39,828 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 00a3709f8e7a19ee64c7444672f69f55: Waiting for close lock at 1733344178124Running coprocessor pre-close hooks at 1733344178124Disabling compacts and flushes for region at 1733344178124Disabling writes for close at 1733344178125 (+1 ms)Obtaining lock to block concurrent updates at 1733344178125Preparing flush snapshotting stores in 00a3709f8e7a19ee64c7444672f69f55 at 1733344178125Finished memstore snapshotting TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733344178126 (+1 ms)Flushing stores of TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55. at 1733344178128 (+2 ms)Flushing 00a3709f8e7a19ee64c7444672f69f55/A: creating writer at 1733344178128Flushing mob file 00a3709f8e7a19ee64c7444672f69f55/A: appending metadata at 1733344178135 (+7 ms)Flushing mob file 00a3709f8e7a19ee64c7444672f69f55/A: closing flushed file at 1733344178135Flushing 00a3709f8e7a19ee64c7444672f69f55/A: appending metadata at 1733344178551 (+416 ms)Flushing 00a3709f8e7a19ee64c7444672f69f55/A: closing flushed file at 1733344178551Flushing 00a3709f8e7a19ee64c7444672f69f55/B: creating writer at 1733344178964 (+413 ms)Flushing 00a3709f8e7a19ee64c7444672f69f55/B: appending metadata at 1733344178968 (+4 ms)Flushing 00a3709f8e7a19ee64c7444672f69f55/B: closing flushed file at 1733344178968Flushing 00a3709f8e7a19ee64c7444672f69f55/C: creating writer at 1733344179384 (+416 ms)Flushing 00a3709f8e7a19ee64c7444672f69f55/C: appending metadata at 1733344179389 (+5 ms)Flushing 00a3709f8e7a19ee64c7444672f69f55/C: closing flushed file at 1733344179389Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@369c0e70: reopening flushed file at 1733344179802 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7be786b7: reopening flushed file at 1733344179810 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21baf7cb: reopening flushed file at 1733344179816 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00a3709f8e7a19ee64c7444672f69f55 in 1696ms, sequenceid=389, compaction requested=true at 1733344179821 (+5 ms)Writing region close event to WAL at 1733344179823 (+2 ms)Running coprocessor post-close hooks at 1733344179828 (+5 ms)Closed at 1733344179828 2024-12-04T20:29:39,830 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:39,831 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=00a3709f8e7a19ee64c7444672f69f55, regionState=CLOSED 2024-12-04T20:29:39,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:29:39,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-04T20:29:39,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 00a3709f8e7a19ee64c7444672f69f55, server=c2ef38372881,45443,1733344054291 in 1.8630 sec 2024-12-04T20:29:39,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-04T20:29:39,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a3709f8e7a19ee64c7444672f69f55, UNASSIGN in 1.8660 sec 2024-12-04T20:29:39,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-04T20:29:39,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8700 sec 2024-12-04T20:29:39,839 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344179839"}]},"ts":"1733344179839"} 2024-12-04T20:29:39,840 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T20:29:39,840 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T20:29:39,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8850 sec 2024-12-04T20:29:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T20:29:40,123 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:40,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T20:29:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:40,129 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T20:29:40,130 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:40,133 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,136 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/recovered.edits] 2024-12-04T20:29:40,140 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/819a5b3e51d240229f0cd53736857884 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/819a5b3e51d240229f0cd53736857884 2024-12-04T20:29:40,142 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b3394a10daab4d0ba93de20e1396552d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/b3394a10daab4d0ba93de20e1396552d 2024-12-04T20:29:40,144 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2ae87be627345ff9c7a9cdb81f054a3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/A/f2ae87be627345ff9c7a9cdb81f054a3 2024-12-04T20:29:40,147 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e1f7bc29b5546338ca3c6367f8c3ecf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/0e1f7bc29b5546338ca3c6367f8c3ecf 2024-12-04T20:29:40,149 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/623a278643b04ac299866b7e511ccb97 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/623a278643b04ac299866b7e511ccb97 2024-12-04T20:29:40,150 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/dd36b52cdd5d49228bf38ae6cd2534c4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/B/dd36b52cdd5d49228bf38ae6cd2534c4 2024-12-04T20:29:40,154 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/5c5cfef0eb6a4a1bbcd3025f7f725959 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/5c5cfef0eb6a4a1bbcd3025f7f725959 2024-12-04T20:29:40,155 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/736f3569894a4cd1a6ab684f7591a473 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/736f3569894a4cd1a6ab684f7591a473 2024-12-04T20:29:40,157 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/dc13242d888d4e67893dca9b490fec5d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/C/dc13242d888d4e67893dca9b490fec5d 2024-12-04T20:29:40,160 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/recovered.edits/392.seqid to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55/recovered.edits/392.seqid 2024-12-04T20:29:40,161 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,161 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T20:29:40,162 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T20:29:40,163 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T20:29:40,168 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040b8032d4069948f1abc49fdece05ee7a_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040b8032d4069948f1abc49fdece05ee7a_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,169 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120421a7dd184f514870b1657b939f2cc392_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120421a7dd184f514870b1657b939f2cc392_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,171 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120432b566a5254049c1a6efa9a2289d6946_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120432b566a5254049c1a6efa9a2289d6946_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,172 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204382e9f4191d646bfa69ddccd45e7d7d6_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204382e9f4191d646bfa69ddccd45e7d7d6_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,173 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120438d662193d684ddabcdd242212bae672_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120438d662193d684ddabcdd242212bae672_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,174 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120438f16edcab124e66a6a19c3fa72698c0_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120438f16edcab124e66a6a19c3fa72698c0_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,175 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043a4c4b2b49344a6caf886a7eb3346a42_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043a4c4b2b49344a6caf886a7eb3346a42_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,175 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120453d021e25bcb495eb492914502917c54_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120453d021e25bcb495eb492914502917c54_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,176 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045a20f9901bc44e37af1c0fb687eb4750_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045a20f9901bc44e37af1c0fb687eb4750_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,177 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e66c80a6123486eaaf35e25984d5566_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e66c80a6123486eaaf35e25984d5566_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,177 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465e95f4f54724f86a9598c5ddce17419_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465e95f4f54724f86a9598c5ddce17419_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,178 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046efbeb1950f34666a985da25da4df03d_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046efbeb1950f34666a985da25da4df03d_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,179 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047bbab141772244c8ba0c5cf66b8f2442_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047bbab141772244c8ba0c5cf66b8f2442_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,180 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204927d5289276640c08ff8185134a1976d_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204927d5289276640c08ff8185134a1976d_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,180 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d9ac2b26f3044111aaa97b3b1b74ca58_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d9ac2b26f3044111aaa97b3b1b74ca58_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,181 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e08c5cb84e454482b0796408becc2841_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e08c5cb84e454482b0796408becc2841_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,182 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e9c3e8b4842c4cfeb810cf8357e36c27_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e9c3e8b4842c4cfeb810cf8357e36c27_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,182 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ff8d1df65a7b4fd8ba387bb87ac4f593_00a3709f8e7a19ee64c7444672f69f55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ff8d1df65a7b4fd8ba387bb87ac4f593_00a3709f8e7a19ee64c7444672f69f55 2024-12-04T20:29:40,183 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T20:29:40,184 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:40,186 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T20:29:40,188 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T20:29:40,189 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:40,189 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T20:29:40,189 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733344180189"}]},"ts":"9223372036854775807"} 2024-12-04T20:29:40,191 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T20:29:40,191 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 00a3709f8e7a19ee64c7444672f69f55, NAME => 'TestAcidGuarantees,,1733344145600.00a3709f8e7a19ee64c7444672f69f55.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T20:29:40,191 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T20:29:40,191 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733344180191"}]},"ts":"9223372036854775807"} 2024-12-04T20:29:40,193 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T20:29:40,193 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:40,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-12-04T20:29:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T20:29:40,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T20:29:40,242 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:40,256 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=503 (was 649), ProcessCount=11 (was 11), AvailableMemoryMB=4677 (was 4795) 2024-12-04T20:29:40,265 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=503, ProcessCount=11, AvailableMemoryMB=4676 2024-12-04T20:29:40,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:29:40,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:29:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T20:29:40,269 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:29:40,269 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:40,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-04T20:29:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T20:29:40,270 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:29:40,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742289_1465 (size=1061) 2024-12-04T20:29:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T20:29:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T20:29:40,680 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:29:40,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742290_1466 (size=53) 2024-12-04T20:29:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T20:29:41,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:29:41,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4bd7564ec5d487ee4345848806acd5a1, disabling compactions & flushes 2024-12-04T20:29:41,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. after waiting 0 ms 2024-12-04T20:29:41,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4bd7564ec5d487ee4345848806acd5a1: Waiting for close lock at 1733344181091Disabling compacts and flushes for region at 1733344181091Disabling writes for close at 1733344181091Writing region close event to WAL at 1733344181091Closed at 1733344181091 2024-12-04T20:29:41,094 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:29:41,094 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733344181094"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733344181094"}]},"ts":"1733344181094"} 2024-12-04T20:29:41,098 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:29:41,100 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:29:41,100 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344181100"}]},"ts":"1733344181100"} 2024-12-04T20:29:41,101 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T20:29:41,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, ASSIGN}] 2024-12-04T20:29:41,103 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, ASSIGN 2024-12-04T20:29:41,103 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, ASSIGN; state=OFFLINE, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=false 2024-12-04T20:29:41,254 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=4bd7564ec5d487ee4345848806acd5a1, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:41,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, ASSIGN because future has completed 2024-12-04T20:29:41,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:29:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T20:29:41,416 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,416 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:29:41,417 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,417 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:29:41,417 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,417 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,418 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,419 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:41,420 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bd7564ec5d487ee4345848806acd5a1 columnFamilyName A 2024-12-04T20:29:41,420 DEBUG [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:41,420 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.HStore(327): Store=4bd7564ec5d487ee4345848806acd5a1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:41,420 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,421 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:41,422 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bd7564ec5d487ee4345848806acd5a1 columnFamilyName B 2024-12-04T20:29:41,422 DEBUG [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:41,422 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.HStore(327): Store=4bd7564ec5d487ee4345848806acd5a1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:41,422 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,424 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:29:41,424 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bd7564ec5d487ee4345848806acd5a1 columnFamilyName C 2024-12-04T20:29:41,424 DEBUG [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:29:41,424 INFO [StoreOpener-4bd7564ec5d487ee4345848806acd5a1-1 {}] regionserver.HStore(327): Store=4bd7564ec5d487ee4345848806acd5a1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:29:41,425 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,425 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,425 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,426 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,426 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,426 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,427 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:29:41,427 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,429 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:29:41,429 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 4bd7564ec5d487ee4345848806acd5a1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67571167, jitterRate=0.006888851523399353}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:29:41,429 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:41,430 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 4bd7564ec5d487ee4345848806acd5a1: Running coprocessor pre-open hook at 1733344181417Writing region info on filesystem at 1733344181417Initializing all the Stores at 1733344181418 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344181418Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344181418Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344181418Cleaning up temporary data from old regions at 1733344181426 (+8 ms)Running coprocessor post-open hooks at 1733344181429 (+3 ms)Region opened successfully at 1733344181430 (+1 ms) 2024-12-04T20:29:41,430 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., pid=114, masterSystemTime=1733344181413 2024-12-04T20:29:41,432 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,432 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:41,432 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=4bd7564ec5d487ee4345848806acd5a1, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:29:41,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:29:41,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-04T20:29:41,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 in 175 msec 2024-12-04T20:29:41,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-04T20:29:41,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, ASSIGN in 335 msec 2024-12-04T20:29:41,440 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:29:41,441 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344181440"}]},"ts":"1733344181440"} 2024-12-04T20:29:41,442 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T20:29:41,443 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:29:41,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-12-04T20:29:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T20:29:42,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:42,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,426 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f020612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c6f8fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,430 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,434 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:29:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,43507,1733344053664 2024-12-04T20:29:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d1108f3 2024-12-04T20:29:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:29:42,437 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:29:42,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,439 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5998c511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@995d0f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31677617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc87e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,449 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586dca86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,452 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a74b8bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e46fee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a33ce4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd291d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,460 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e12e77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,462 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c2d92dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,463 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262e50c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,465 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d249d14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5e0af2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a041ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2558e37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,472 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649fa630, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,474 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16480600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:29:42,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:29:42,476 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:29:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:29:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:29:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7566c5fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:29:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:29:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:29:42,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:29:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@236c3c53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:29:42,478 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,479 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,479 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,479 DEBUG [Thread-2083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:42,479 DEBUG [Thread-2089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,480 DEBUG [Thread-2102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,481 DEBUG [Thread-2083 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,481 DEBUG [Thread-2091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-04T20:29:42,482 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,482 DEBUG [Thread-2087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T20:29:42,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,483 DEBUG [Thread-2100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:29:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,483 DEBUG [Thread-2083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:29:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,485 DEBUG [Thread-2098 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:29:42,485 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:42,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:29:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:42,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2] 2024-12-04T20:29:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:42,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:29:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:42,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8b2d9d929f284439b334f895a2febf25 is 50, key is test_row_0/A:col10/1733344182487/Put/seqid=0 2024-12-04T20:29:42,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742291_1467 (size=12001) 2024-12-04T20:29:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344182503, max timestamp=1733344182557Num uniques 859; 2024-12-04T20:29:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344182503, max timestamp=1733344182557Num uniques 859; 2024-12-04T20:29:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344182503, max timestamp=1733344182557Num uniques 859; 2024-12-04T20:29:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344242579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344242580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344242581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344242582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344242583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T20:29:42,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T20:29:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:42,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344242691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344242692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344242692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344242701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344242701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T20:29:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:42,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T20:29:42,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344242902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344242902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344242902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8b2d9d929f284439b334f895a2febf25 2024-12-04T20:29:42,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344242911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344242911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:42,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/f772a078ae704c3e95945d3ac02eb62e is 50, key is test_row_0/B:col10/1733344182487/Put/seqid=0 2024-12-04T20:29:42,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742292_1468 (size=12001) 2024-12-04T20:29:42,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T20:29:42,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:42,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:42,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:42,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:43,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T20:29:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:43,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T20:29:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344243212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344243212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344243212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344243221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344243221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T20:29:43,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:43,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:43,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:43,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:43,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/f772a078ae704c3e95945d3ac02eb62e 2024-12-04T20:29:43,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/fc90c082257d4c6d9a9d381e2efd34ea is 50, key is test_row_0/C:col10/1733344182487/Put/seqid=0 2024-12-04T20:29:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742293_1469 (size=12001) 2024-12-04T20:29:43,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/fc90c082257d4c6d9a9d381e2efd34ea 2024-12-04T20:29:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8b2d9d929f284439b334f895a2febf25 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8b2d9d929f284439b334f895a2febf25 2024-12-04T20:29:43,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8b2d9d929f284439b334f895a2febf25, entries=150, sequenceid=11, filesize=11.7 K 2024-12-04T20:29:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/f772a078ae704c3e95945d3ac02eb62e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/f772a078ae704c3e95945d3ac02eb62e 2024-12-04T20:29:43,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/f772a078ae704c3e95945d3ac02eb62e, entries=150, sequenceid=11, filesize=11.7 K 2024-12-04T20:29:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/fc90c082257d4c6d9a9d381e2efd34ea as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/fc90c082257d4c6d9a9d381e2efd34ea 2024-12-04T20:29:43,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/fc90c082257d4c6d9a9d381e2efd34ea, entries=150, sequenceid=11, filesize=11.7 K 2024-12-04T20:29:43,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4bd7564ec5d487ee4345848806acd5a1 in 881ms, sequenceid=11, compaction requested=false 2024-12-04T20:29:43,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:43,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T20:29:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:43,405 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T20:29:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:43,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/629fa9dd6fcd4c2d98a3b72834513cb2 is 50, key is test_row_0/A:col10/1733344182581/Put/seqid=0 2024-12-04T20:29:43,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742294_1470 (size=12001) 2024-12-04T20:29:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T20:29:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:43,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344243737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344243737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344243738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344243740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344243741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,814 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/629fa9dd6fcd4c2d98a3b72834513cb2 2024-12-04T20:29:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/12a00e112f6440a8aa3e19d6084bc9bc is 50, key is test_row_0/B:col10/1733344182581/Put/seqid=0 2024-12-04T20:29:43,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742295_1471 (size=12001) 2024-12-04T20:29:43,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344243851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344243851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344243852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344243852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344243852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344244061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344244062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344244062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344244062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344244062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/12a00e112f6440a8aa3e19d6084bc9bc 2024-12-04T20:29:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/f2139123b75243bc8b697e72b52458a8 is 50, key is test_row_0/C:col10/1733344182581/Put/seqid=0 2024-12-04T20:29:44,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742296_1472 (size=12001) 2024-12-04T20:29:44,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344244371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344244371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344244381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344244382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344244382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T20:29:44,634 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/f2139123b75243bc8b697e72b52458a8 2024-12-04T20:29:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/629fa9dd6fcd4c2d98a3b72834513cb2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/629fa9dd6fcd4c2d98a3b72834513cb2 2024-12-04T20:29:44,640 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/629fa9dd6fcd4c2d98a3b72834513cb2, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T20:29:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/12a00e112f6440a8aa3e19d6084bc9bc as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/12a00e112f6440a8aa3e19d6084bc9bc 2024-12-04T20:29:44,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/12a00e112f6440a8aa3e19d6084bc9bc, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T20:29:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/f2139123b75243bc8b697e72b52458a8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f2139123b75243bc8b697e72b52458a8 2024-12-04T20:29:44,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f2139123b75243bc8b697e72b52458a8, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T20:29:44,647 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 4bd7564ec5d487ee4345848806acd5a1 in 1242ms, sequenceid=46, compaction requested=false 2024-12-04T20:29:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-04T20:29:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-04T20:29:44,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-04T20:29:44,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1600 sec 2024-12-04T20:29:44,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.1710 sec 2024-12-04T20:29:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:44,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T20:29:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:44,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/62241ff5807f41b7a20f01f0461bcda7 is 50, key is test_row_0/A:col10/1733344184882/Put/seqid=0 2024-12-04T20:29:44,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742297_1473 (size=14341) 2024-12-04T20:29:44,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344244909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344244910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344184903, max timestamp=1733344184918Num uniques 859; 2024-12-04T20:29:44,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344184903, max timestamp=1733344184918Num uniques 859; 2024-12-04T20:29:44,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344184903, max timestamp=1733344184918Num uniques 859; 2024-12-04T20:29:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344244917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:44,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:44,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344244958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:44,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344244960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344245021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344245031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344245031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344245071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344245071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344245231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344245241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344245242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344245281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344245281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/62241ff5807f41b7a20f01f0461bcda7 2024-12-04T20:29:45,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/5822389f4f2c489da379f986c1a4dea0 is 50, key is test_row_0/B:col10/1733344184882/Put/seqid=0 2024-12-04T20:29:45,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742298_1474 (size=12001) 2024-12-04T20:29:45,343 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:29:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344245541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344245551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344245562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344245591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344245591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:45,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/5822389f4f2c489da379f986c1a4dea0 2024-12-04T20:29:45,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/0b48f9a1d7cb431596bb265eaa178743 is 50, key is test_row_0/C:col10/1733344184882/Put/seqid=0 2024-12-04T20:29:45,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742299_1475 (size=12001) 2024-12-04T20:29:45,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/0b48f9a1d7cb431596bb265eaa178743 2024-12-04T20:29:45,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/62241ff5807f41b7a20f01f0461bcda7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/62241ff5807f41b7a20f01f0461bcda7 2024-12-04T20:29:45,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/62241ff5807f41b7a20f01f0461bcda7, entries=200, sequenceid=57, filesize=14.0 K 2024-12-04T20:29:45,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/5822389f4f2c489da379f986c1a4dea0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/5822389f4f2c489da379f986c1a4dea0 2024-12-04T20:29:45,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/5822389f4f2c489da379f986c1a4dea0, entries=150, sequenceid=57, filesize=11.7 K 2024-12-04T20:29:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/0b48f9a1d7cb431596bb265eaa178743 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/0b48f9a1d7cb431596bb265eaa178743 2024-12-04T20:29:45,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/0b48f9a1d7cb431596bb265eaa178743, entries=150, sequenceid=57, filesize=11.7 K 2024-12-04T20:29:45,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4bd7564ec5d487ee4345848806acd5a1 in 843ms, sequenceid=57, compaction requested=true 2024-12-04T20:29:45,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:45,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:45,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:45,727 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:45,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:45,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:45,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:45,727 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:45,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/B is initiating minor compaction (all files) 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/A is initiating minor compaction (all files) 2024-12-04T20:29:45,728 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/B in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:45,728 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/A in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:45,728 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8b2d9d929f284439b334f895a2febf25, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/629fa9dd6fcd4c2d98a3b72834513cb2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/62241ff5807f41b7a20f01f0461bcda7] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=37.4 K 2024-12-04T20:29:45,728 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/f772a078ae704c3e95945d3ac02eb62e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/12a00e112f6440a8aa3e19d6084bc9bc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/5822389f4f2c489da379f986c1a4dea0] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=35.2 K 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b2d9d929f284439b334f895a2febf25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733344182487 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f772a078ae704c3e95945d3ac02eb62e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733344182487 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 629fa9dd6fcd4c2d98a3b72834513cb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733344182569 2024-12-04T20:29:45,728 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 12a00e112f6440a8aa3e19d6084bc9bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733344182569 2024-12-04T20:29:45,729 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62241ff5807f41b7a20f01f0461bcda7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733344183737 2024-12-04T20:29:45,729 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 5822389f4f2c489da379f986c1a4dea0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733344183737 2024-12-04T20:29:45,736 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:45,737 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/eb4a27bb282e45df87c982f74c38db55 is 50, key is test_row_0/B:col10/1733344184882/Put/seqid=0 2024-12-04T20:29:45,738 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:45,738 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/00c2187bf83040b6b127b85924486c4b is 50, key is test_row_0/A:col10/1733344184882/Put/seqid=0 2024-12-04T20:29:45,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742301_1477 (size=12104) 2024-12-04T20:29:45,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742300_1476 (size=12104) 2024-12-04T20:29:45,756 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/00c2187bf83040b6b127b85924486c4b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/00c2187bf83040b6b127b85924486c4b 2024-12-04T20:29:45,757 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/eb4a27bb282e45df87c982f74c38db55 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/eb4a27bb282e45df87c982f74c38db55 2024-12-04T20:29:45,760 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/B of 4bd7564ec5d487ee4345848806acd5a1 into eb4a27bb282e45df87c982f74c38db55(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:45,760 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:45,760 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/B, priority=13, startTime=1733344185727; duration=0sec 2024-12-04T20:29:45,760 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:45,760 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:B 2024-12-04T20:29:45,760 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:45,761 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/A of 4bd7564ec5d487ee4345848806acd5a1 into 00c2187bf83040b6b127b85924486c4b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:45,761 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:45,761 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/A, priority=13, startTime=1733344185727; duration=0sec 2024-12-04T20:29:45,761 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:45,761 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:45,761 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:A 2024-12-04T20:29:45,761 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/C is initiating minor compaction (all files) 2024-12-04T20:29:45,761 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/C in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:45,761 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/fc90c082257d4c6d9a9d381e2efd34ea, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f2139123b75243bc8b697e72b52458a8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/0b48f9a1d7cb431596bb265eaa178743] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=35.2 K 2024-12-04T20:29:45,761 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting fc90c082257d4c6d9a9d381e2efd34ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733344182487 2024-12-04T20:29:45,761 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f2139123b75243bc8b697e72b52458a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733344182569 2024-12-04T20:29:45,762 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b48f9a1d7cb431596bb265eaa178743, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733344183737 2024-12-04T20:29:45,767 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:45,767 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/717c2a4d1c744dcb952d2f59b40f8ab3 is 50, key is test_row_0/C:col10/1733344184882/Put/seqid=0 2024-12-04T20:29:45,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742302_1478 (size=12104) 2024-12-04T20:29:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:46,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T20:29:46,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:46,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/74ef5019b5fc4bdf95e923a4638ef93b is 50, key is test_row_0/A:col10/1733344184959/Put/seqid=0 2024-12-04T20:29:46,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742303_1479 (size=14341) 2024-12-04T20:29:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344246074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344246081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344246081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344246111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344246111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,175 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/717c2a4d1c744dcb952d2f59b40f8ab3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/717c2a4d1c744dcb952d2f59b40f8ab3 2024-12-04T20:29:46,179 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/C of 4bd7564ec5d487ee4345848806acd5a1 into 717c2a4d1c744dcb952d2f59b40f8ab3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:46,179 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:46,179 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/C, priority=13, startTime=1733344185727; duration=0sec 2024-12-04T20:29:46,179 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:46,179 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:C 2024-12-04T20:29:46,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344246191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344246191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344246191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344246402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344246402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344246402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/74ef5019b5fc4bdf95e923a4638ef93b 2024-12-04T20:29:46,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/9ae25021ea324d299e23f0bfc4b01c0f is 50, key is test_row_0/B:col10/1733344184959/Put/seqid=0 2024-12-04T20:29:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742304_1480 (size=12001) 2024-12-04T20:29:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T20:29:46,642 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:46,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-04T20:29:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T20:29:46,644 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:46,644 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:46,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:46,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344246711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344246711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344246711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T20:29:46,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T20:29:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:46,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:46,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/9ae25021ea324d299e23f0bfc4b01c0f 2024-12-04T20:29:46,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/89a1eb30e7a6438da09da98a5936e138 is 50, key is test_row_0/C:col10/1733344184959/Put/seqid=0 2024-12-04T20:29:46,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742305_1481 (size=12001) 2024-12-04T20:29:46,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T20:29:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:46,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T20:29:47,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T20:29:47,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:47,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:47,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:47,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:47,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:47,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344247122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:47,128 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 4bd7564ec5d487ee4345848806acd5a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:47,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344247131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:47,137 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 4bd7564ec5d487ee4345848806acd5a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:47,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344247222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:47,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344247222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:47,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344247222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:47,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T20:29:47,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:47,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:47,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:47,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:47,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T20:29:47,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/89a1eb30e7a6438da09da98a5936e138 2024-12-04T20:29:47,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/74ef5019b5fc4bdf95e923a4638ef93b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/74ef5019b5fc4bdf95e923a4638ef93b 2024-12-04T20:29:47,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/74ef5019b5fc4bdf95e923a4638ef93b, entries=200, sequenceid=94, filesize=14.0 K 2024-12-04T20:29:47,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/9ae25021ea324d299e23f0bfc4b01c0f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/9ae25021ea324d299e23f0bfc4b01c0f 2024-12-04T20:29:47,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/9ae25021ea324d299e23f0bfc4b01c0f, entries=150, sequenceid=94, filesize=11.7 K 2024-12-04T20:29:47,291 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-04T20:29:47,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/89a1eb30e7a6438da09da98a5936e138 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/89a1eb30e7a6438da09da98a5936e138 2024-12-04T20:29:47,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/89a1eb30e7a6438da09da98a5936e138, entries=150, sequenceid=94, filesize=11.7 K 2024-12-04T20:29:47,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 4bd7564ec5d487ee4345848806acd5a1 in 1238ms, sequenceid=94, compaction requested=false 2024-12-04T20:29:47,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:47,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T20:29:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:47,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:29:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:47,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/d5a40554ee184562bf8385aaf5ae091e is 50, key is test_row_0/A:col10/1733344186068/Put/seqid=0 2024-12-04T20:29:47,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742306_1482 (size=12001) 2024-12-04T20:29:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T20:29:47,816 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/d5a40554ee184562bf8385aaf5ae091e 2024-12-04T20:29:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/6a2baf291c454e72a382f0f57ed3612f is 50, key is test_row_0/B:col10/1733344186068/Put/seqid=0 2024-12-04T20:29:47,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742307_1483 (size=12001) 2024-12-04T20:29:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:48,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:48,259 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/6a2baf291c454e72a382f0f57ed3612f 2024-12-04T20:29:48,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/9fe557fd6a2a4deab5e744b850f0b74b is 50, key is test_row_0/C:col10/1733344186068/Put/seqid=0 2024-12-04T20:29:48,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742308_1484 (size=12001) 2024-12-04T20:29:48,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344188254, max timestamp=1733344188270Num uniques 859; 2024-12-04T20:29:48,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344188254, max timestamp=1733344188270Num uniques 859; 2024-12-04T20:29:48,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344188254, max timestamp=1733344188270Num uniques 859; 2024-12-04T20:29:48,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:48,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:48,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:48,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344248304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344248305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344248311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344248422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344248422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344248422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344248631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344248631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344248632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,667 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/9fe557fd6a2a4deab5e744b850f0b74b 2024-12-04T20:29:48,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/d5a40554ee184562bf8385aaf5ae091e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d5a40554ee184562bf8385aaf5ae091e 2024-12-04T20:29:48,673 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d5a40554ee184562bf8385aaf5ae091e, entries=150, sequenceid=105, filesize=11.7 K 2024-12-04T20:29:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/6a2baf291c454e72a382f0f57ed3612f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6a2baf291c454e72a382f0f57ed3612f 2024-12-04T20:29:48,676 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6a2baf291c454e72a382f0f57ed3612f, entries=150, sequenceid=105, filesize=11.7 K 2024-12-04T20:29:48,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/9fe557fd6a2a4deab5e744b850f0b74b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9fe557fd6a2a4deab5e744b850f0b74b 2024-12-04T20:29:48,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9fe557fd6a2a4deab5e744b850f0b74b, entries=150, sequenceid=105, filesize=11.7 K 2024-12-04T20:29:48,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4bd7564ec5d487ee4345848806acd5a1 in 1271ms, sequenceid=105, compaction requested=true 2024-12-04T20:29:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-04T20:29:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-04T20:29:48,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-04T20:29:48,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0360 sec 2024-12-04T20:29:48,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.0410 sec 2024-12-04T20:29:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T20:29:48,791 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:48,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-04T20:29:48,793 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T20:29:48,794 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:48,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T20:29:48,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T20:29:48,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:48,946 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T20:29:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:48,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/1e01669b2a09444dab1b2967a41374a9 is 50, key is test_row_0/A:col10/1733344188295/Put/seqid=0 2024-12-04T20:29:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742309_1485 (size=14541) 2024-12-04T20:29:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344248962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344248962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344248973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344249082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344249082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344249091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T20:29:49,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344249142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,149 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 4bd7564ec5d487ee4345848806acd5a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:49,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344249151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,158 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 4bd7564ec5d487ee4345848806acd5a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344249291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344249291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344249301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/1e01669b2a09444dab1b2967a41374a9 2024-12-04T20:29:49,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a05ccf2e193e42228c023b3cfeaae633 is 50, key is test_row_0/B:col10/1733344188295/Put/seqid=0 2024-12-04T20:29:49,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742310_1486 (size=12151) 2024-12-04T20:29:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T20:29:49,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344249602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344249612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344249612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:49,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a05ccf2e193e42228c023b3cfeaae633 2024-12-04T20:29:49,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e6b68f98b91f47c1b6d0204e3d95e6bb is 50, key is test_row_0/C:col10/1733344188295/Put/seqid=0 2024-12-04T20:29:49,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742311_1487 (size=12151) 2024-12-04T20:29:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T20:29:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344250122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:50,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344250133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:50,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344250133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:50,172 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e6b68f98b91f47c1b6d0204e3d95e6bb 2024-12-04T20:29:50,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/1e01669b2a09444dab1b2967a41374a9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1e01669b2a09444dab1b2967a41374a9 2024-12-04T20:29:50,187 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1e01669b2a09444dab1b2967a41374a9, entries=200, sequenceid=141, filesize=14.2 K 2024-12-04T20:29:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a05ccf2e193e42228c023b3cfeaae633 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a05ccf2e193e42228c023b3cfeaae633 2024-12-04T20:29:50,193 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a05ccf2e193e42228c023b3cfeaae633, entries=150, sequenceid=141, filesize=11.9 K 2024-12-04T20:29:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e6b68f98b91f47c1b6d0204e3d95e6bb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e6b68f98b91f47c1b6d0204e3d95e6bb 2024-12-04T20:29:50,196 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e6b68f98b91f47c1b6d0204e3d95e6bb, entries=150, sequenceid=141, filesize=11.9 K 2024-12-04T20:29:50,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 4bd7564ec5d487ee4345848806acd5a1 in 1252ms, sequenceid=141, compaction requested=true 2024-12-04T20:29:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-04T20:29:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-04T20:29:50,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-04T20:29:50,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-12-04T20:29:50,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4090 sec 2024-12-04T20:29:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T20:29:50,942 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:50,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-04T20:29:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T20:29:50,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:50,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:50,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T20:29:51,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T20:29:51,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:51,096 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T20:29:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8af2677fa2704499bc6a3e419aa8014c is 50, key is test_row_0/A:col10/1733344188959/Put/seqid=0 2024-12-04T20:29:51,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742312_1488 (size=12151) 2024-12-04T20:29:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:51,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:51,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344191190, max timestamp=1733344191218Num uniques 859; 2024-12-04T20:29:51,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:51,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344191190, max timestamp=1733344191218Num uniques 859; 2024-12-04T20:29:51,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:51,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344191190, max timestamp=1733344191218Num uniques 859; 2024-12-04T20:29:51,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T20:29:51,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344191252, max timestamp=1733344191285Num uniques 1068; 2024-12-04T20:29:51,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:29:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344191252, max timestamp=1733344191285Num uniques 1068; 2024-12-04T20:29:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:29:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733344191252, max timestamp=1733344191285Num uniques 1068; 2024-12-04T20:29:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T20:29:51,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344251347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344251349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344251356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344251472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344251472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344251472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,507 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8af2677fa2704499bc6a3e419aa8014c 2024-12-04T20:29:51,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/d22e52b11fd844daba39263fbd1e6d52 is 50, key is test_row_0/B:col10/1733344188959/Put/seqid=0 2024-12-04T20:29:51,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742313_1489 (size=12151) 2024-12-04T20:29:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T20:29:51,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344251681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344251683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344251683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:51,915 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/d22e52b11fd844daba39263fbd1e6d52 2024-12-04T20:29:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/c7aa5b38ea774115a4180420c9612ba3 is 50, key is test_row_0/C:col10/1733344188959/Put/seqid=0 2024-12-04T20:29:51,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742314_1490 (size=12151) 2024-12-04T20:29:51,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344251991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344252001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344252001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T20:29:52,326 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/c7aa5b38ea774115a4180420c9612ba3 2024-12-04T20:29:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8af2677fa2704499bc6a3e419aa8014c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8af2677fa2704499bc6a3e419aa8014c 2024-12-04T20:29:52,332 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8af2677fa2704499bc6a3e419aa8014c, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T20:29:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/d22e52b11fd844daba39263fbd1e6d52 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d22e52b11fd844daba39263fbd1e6d52 2024-12-04T20:29:52,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d22e52b11fd844daba39263fbd1e6d52, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T20:29:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/c7aa5b38ea774115a4180420c9612ba3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c7aa5b38ea774115a4180420c9612ba3 2024-12-04T20:29:52,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c7aa5b38ea774115a4180420c9612ba3, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T20:29:52,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 4bd7564ec5d487ee4345848806acd5a1 in 1242ms, sequenceid=150, compaction requested=true 2024-12-04T20:29:52,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:52,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:52,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-04T20:29:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-04T20:29:52,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-04T20:29:52,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3950 sec 2024-12-04T20:29:52,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3990 sec 2024-12-04T20:29:52,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-04T20:29:52,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:52,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:29:52,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:52,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:29:52,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:52,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T20:29:52,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/ad2422a9ab314978b64dcefb08c55d52 is 50, key is test_row_0/A:col10/1733344191354/Put/seqid=0 2024-12-04T20:29:52,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742315_1491 (size=14541) 2024-12-04T20:29:52,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344252529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344252530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344252535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344252651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344252651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344252651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344252861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344252861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344252861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:52,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/ad2422a9ab314978b64dcefb08c55d52 2024-12-04T20:29:52,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/d48966c17e064304a76032ddfdde1d14 is 50, key is test_row_0/B:col10/1733344191354/Put/seqid=0 2024-12-04T20:29:52,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742316_1492 (size=12151) 2024-12-04T20:29:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T20:29:53,091 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:53,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-04T20:29:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T20:29:53,094 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:53,094 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:53,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:53,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344253172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344253172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54398 deadline: 1733344253172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344253172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,184 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in 4bd7564ec5d487ee4345848806acd5a1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T20:29:53,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54420 deadline: 1733344253191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,203 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_1' in 4bd7564ec5d487ee4345848806acd5a1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:29:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T20:29:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/d48966c17e064304a76032ddfdde1d14 2024-12-04T20:29:53,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/68604435b32749f9a1561d8c075a6afc is 50, key is test_row_0/C:col10/1733344191354/Put/seqid=0 2024-12-04T20:29:53,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742317_1493 (size=12151) 2024-12-04T20:29:53,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T20:29:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:53,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T20:29:53,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T20:29:53,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344253691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344253691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344253691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:53,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T20:29:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T20:29:53,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/68604435b32749f9a1561d8c075a6afc 2024-12-04T20:29:53,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/ad2422a9ab314978b64dcefb08c55d52 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/ad2422a9ab314978b64dcefb08c55d52 2024-12-04T20:29:53,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/ad2422a9ab314978b64dcefb08c55d52, entries=200, sequenceid=205, filesize=14.2 K 2024-12-04T20:29:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/d48966c17e064304a76032ddfdde1d14 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d48966c17e064304a76032ddfdde1d14 2024-12-04T20:29:53,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d48966c17e064304a76032ddfdde1d14, entries=150, sequenceid=205, filesize=11.9 K 2024-12-04T20:29:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/68604435b32749f9a1561d8c075a6afc as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68604435b32749f9a1561d8c075a6afc 2024-12-04T20:29:53,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68604435b32749f9a1561d8c075a6afc, entries=150, sequenceid=205, filesize=11.9 K 2024-12-04T20:29:53,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 4bd7564ec5d487ee4345848806acd5a1 in 1304ms, sequenceid=205, compaction requested=true 2024-12-04T20:29:53,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:53,821 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-04T20:29:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:53,821 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-04T20:29:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:53,829 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72559 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-04T20:29:53,829 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/B is initiating minor compaction (all files) 2024-12-04T20:29:53,829 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 79679 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-04T20:29:53,829 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/B in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,829 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/A is initiating minor compaction (all files) 2024-12-04T20:29:53,829 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/A in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,829 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/eb4a27bb282e45df87c982f74c38db55, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/9ae25021ea324d299e23f0bfc4b01c0f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6a2baf291c454e72a382f0f57ed3612f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a05ccf2e193e42228c023b3cfeaae633, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d22e52b11fd844daba39263fbd1e6d52, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d48966c17e064304a76032ddfdde1d14] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=70.9 K 2024-12-04T20:29:53,830 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/00c2187bf83040b6b127b85924486c4b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/74ef5019b5fc4bdf95e923a4638ef93b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d5a40554ee184562bf8385aaf5ae091e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1e01669b2a09444dab1b2967a41374a9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8af2677fa2704499bc6a3e419aa8014c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/ad2422a9ab314978b64dcefb08c55d52] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=77.8 K 2024-12-04T20:29:53,830 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting eb4a27bb282e45df87c982f74c38db55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733344183737 2024-12-04T20:29:53,830 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c2187bf83040b6b127b85924486c4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733344183737 2024-12-04T20:29:53,830 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae25021ea324d299e23f0bfc4b01c0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733344184952 2024-12-04T20:29:53,830 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ef5019b5fc4bdf95e923a4638ef93b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733344184948 2024-12-04T20:29:53,830 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2baf291c454e72a382f0f57ed3612f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733344186068 2024-12-04T20:29:53,830 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a40554ee184562bf8385aaf5ae091e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733344186068 2024-12-04T20:29:53,831 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a05ccf2e193e42228c023b3cfeaae633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733344188295 2024-12-04T20:29:53,831 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d22e52b11fd844daba39263fbd1e6d52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733344188959 2024-12-04T20:29:53,831 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e01669b2a09444dab1b2967a41374a9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733344188295 2024-12-04T20:29:53,831 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d48966c17e064304a76032ddfdde1d14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344191337 2024-12-04T20:29:53,831 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8af2677fa2704499bc6a3e419aa8014c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733344188959 2024-12-04T20:29:53,832 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad2422a9ab314978b64dcefb08c55d52, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344191337 2024-12-04T20:29:53,853 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#B#compaction#414 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:53,854 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/826ff7027a0d49adb9ecfbbdd0f149a8 is 50, key is test_row_0/B:col10/1733344191354/Put/seqid=0 2024-12-04T20:29:53,854 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:53,855 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/605f5219bb64409db8919470308a6591 is 50, key is test_row_0/A:col10/1733344191354/Put/seqid=0 2024-12-04T20:29:53,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T20:29:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,861 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T20:29:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:53,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742318_1494 (size=12459) 2024-12-04T20:29:53,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742319_1495 (size=12459) 2024-12-04T20:29:53,898 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/605f5219bb64409db8919470308a6591 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/605f5219bb64409db8919470308a6591 2024-12-04T20:29:53,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/d21ef4b39c8e4231a815ccc1a8c2c8b9 is 50, key is test_row_1/A:col10/1733344192534/Put/seqid=0 2024-12-04T20:29:53,908 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/A of 4bd7564ec5d487ee4345848806acd5a1 into 605f5219bb64409db8919470308a6591(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:53,908 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:53,908 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/A, priority=10, startTime=1733344193820; duration=0sec 2024-12-04T20:29:53,908 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:53,908 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:A 2024-12-04T20:29:53,909 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-04T20:29:53,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742320_1496 (size=9757) 2024-12-04T20:29:53,910 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72559 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-04T20:29:53,910 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/C is initiating minor compaction (all files) 2024-12-04T20:29:53,910 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/C in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:53,910 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/717c2a4d1c744dcb952d2f59b40f8ab3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/89a1eb30e7a6438da09da98a5936e138, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9fe557fd6a2a4deab5e744b850f0b74b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e6b68f98b91f47c1b6d0204e3d95e6bb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c7aa5b38ea774115a4180420c9612ba3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68604435b32749f9a1561d8c075a6afc] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=70.9 K 2024-12-04T20:29:53,910 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717c2a4d1c744dcb952d2f59b40f8ab3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733344183737 2024-12-04T20:29:53,911 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89a1eb30e7a6438da09da98a5936e138, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733344184952 2024-12-04T20:29:53,911 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe557fd6a2a4deab5e744b850f0b74b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733344186068 2024-12-04T20:29:53,911 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b68f98b91f47c1b6d0204e3d95e6bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733344188295 2024-12-04T20:29:53,911 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7aa5b38ea774115a4180420c9612ba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733344188959 2024-12-04T20:29:53,912 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68604435b32749f9a1561d8c075a6afc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344191337 2024-12-04T20:29:53,930 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#C#compaction#417 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:53,931 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/ec34d583981e480f8f74e61829fab1f8 is 50, key is test_row_0/C:col10/1733344191354/Put/seqid=0 2024-12-04T20:29:53,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742321_1497 (size=12459) 2024-12-04T20:29:53,958 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/ec34d583981e480f8f74e61829fab1f8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/ec34d583981e480f8f74e61829fab1f8 2024-12-04T20:29:53,966 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/C of 4bd7564ec5d487ee4345848806acd5a1 into ec34d583981e480f8f74e61829fab1f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:53,966 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:53,966 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/C, priority=10, startTime=1733344193821; duration=0sec 2024-12-04T20:29:53,966 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:53,966 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:C 2024-12-04T20:29:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T20:29:54,291 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/826ff7027a0d49adb9ecfbbdd0f149a8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/826ff7027a0d49adb9ecfbbdd0f149a8 2024-12-04T20:29:54,299 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/B of 4bd7564ec5d487ee4345848806acd5a1 into 826ff7027a0d49adb9ecfbbdd0f149a8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:54,299 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:54,299 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/B, priority=10, startTime=1733344193821; duration=0sec 2024-12-04T20:29:54,299 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:54,299 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:B 2024-12-04T20:29:54,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/d21ef4b39c8e4231a815ccc1a8c2c8b9 2024-12-04T20:29:54,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/bf1d579f534c497290abaf3c4b76e1f0 is 50, key is test_row_1/B:col10/1733344192534/Put/seqid=0 2024-12-04T20:29:54,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742322_1498 (size=9757) 2024-12-04T20:29:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:54,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:54,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344194721, max timestamp=1733344194744Num uniques 859; 2024-12-04T20:29:54,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344194721, max timestamp=1733344194744Num uniques 859; 2024-12-04T20:29:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733344194721, max timestamp=1733344194744Num uniques 859; 2024-12-04T20:29:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T20:29:54,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/bf1d579f534c497290abaf3c4b76e1f0 2024-12-04T20:29:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/7c23a07a92704d43981dc7f2fe39043f is 50, key is test_row_1/C:col10/1733344192534/Put/seqid=0 2024-12-04T20:29:54,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742323_1499 (size=9757) 2024-12-04T20:29:54,770 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/7c23a07a92704d43981dc7f2fe39043f 2024-12-04T20:29:54,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/d21ef4b39c8e4231a815ccc1a8c2c8b9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d21ef4b39c8e4231a815ccc1a8c2c8b9 2024-12-04T20:29:54,775 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d21ef4b39c8e4231a815ccc1a8c2c8b9, entries=100, sequenceid=213, filesize=9.5 K 2024-12-04T20:29:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/bf1d579f534c497290abaf3c4b76e1f0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/bf1d579f534c497290abaf3c4b76e1f0 2024-12-04T20:29:54,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/bf1d579f534c497290abaf3c4b76e1f0, entries=100, sequenceid=213, filesize=9.5 K 2024-12-04T20:29:54,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/7c23a07a92704d43981dc7f2fe39043f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/7c23a07a92704d43981dc7f2fe39043f 2024-12-04T20:29:54,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/7c23a07a92704d43981dc7f2fe39043f, entries=100, sequenceid=213, filesize=9.5 K 2024-12-04T20:29:54,786 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=134.18 KB/137400 for 4bd7564ec5d487ee4345848806acd5a1 in 925ms, sequenceid=213, compaction requested=false 2024-12-04T20:29:54,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:54,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:54,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-04T20:29:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-04T20:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:54,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-04T20:29:54,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:54,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-04T20:29:54,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6930 sec 2024-12-04T20:29:54,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6980 sec 2024-12-04T20:29:54,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/80a9a8a5202f4be7a9f84313f7b0051c is 50, key is test_row_0/A:col10/1733344194779/Put/seqid=0 2024-12-04T20:29:54,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742324_1500 (size=14541) 2024-12-04T20:29:54,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344254797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344254805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:54,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344254807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344254911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344254911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:54,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344254922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344255121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344255121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344255132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/80a9a8a5202f4be7a9f84313f7b0051c 2024-12-04T20:29:55,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/ea7ac08b11364f428821440c9654bd62 is 50, key is test_row_0/B:col10/1733344194779/Put/seqid=0 2024-12-04T20:29:55,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742325_1501 (size=12151) 2024-12-04T20:29:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T20:29:55,242 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:55,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-04T20:29:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T20:29:55,243 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:55,244 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:55,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T20:29:55,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T20:29:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344255431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344255431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344255441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T20:29:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T20:29:55,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/ea7ac08b11364f428821440c9654bd62 2024-12-04T20:29:55,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/9ef3d38967154b47a8e7e8b40bbcf543 is 50, key is test_row_0/C:col10/1733344194779/Put/seqid=0 2024-12-04T20:29:55,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742326_1502 (size=12151) 2024-12-04T20:29:55,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T20:29:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T20:29:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:55,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T20:29:55,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344255941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344255941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:55,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344255962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:56,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T20:29:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:56,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:56,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:29:56,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/9ef3d38967154b47a8e7e8b40bbcf543 2024-12-04T20:29:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/80a9a8a5202f4be7a9f84313f7b0051c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/80a9a8a5202f4be7a9f84313f7b0051c 2024-12-04T20:29:56,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/80a9a8a5202f4be7a9f84313f7b0051c, entries=200, sequenceid=253, filesize=14.2 K 2024-12-04T20:29:56,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/ea7ac08b11364f428821440c9654bd62 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/ea7ac08b11364f428821440c9654bd62 2024-12-04T20:29:56,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/ea7ac08b11364f428821440c9654bd62, entries=150, sequenceid=253, filesize=11.9 K 2024-12-04T20:29:56,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/9ef3d38967154b47a8e7e8b40bbcf543 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9ef3d38967154b47a8e7e8b40bbcf543 2024-12-04T20:29:56,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9ef3d38967154b47a8e7e8b40bbcf543, entries=150, sequenceid=253, filesize=11.9 K 2024-12-04T20:29:56,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 4bd7564ec5d487ee4345848806acd5a1 in 1265ms, sequenceid=253, compaction requested=true 2024-12-04T20:29:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:56,054 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:56,054 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:56,058 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:56,058 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/B is initiating minor compaction (all files) 2024-12-04T20:29:56,058 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/B in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:56,058 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/826ff7027a0d49adb9ecfbbdd0f149a8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/bf1d579f534c497290abaf3c4b76e1f0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/ea7ac08b11364f428821440c9654bd62] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=33.6 K 2024-12-04T20:29:56,059 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:56,059 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/A is initiating minor compaction (all files) 2024-12-04T20:29:56,059 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/A in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:56,059 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/605f5219bb64409db8919470308a6591, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d21ef4b39c8e4231a815ccc1a8c2c8b9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/80a9a8a5202f4be7a9f84313f7b0051c] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=35.9 K 2024-12-04T20:29:56,059 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 826ff7027a0d49adb9ecfbbdd0f149a8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344191337 2024-12-04T20:29:56,059 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 605f5219bb64409db8919470308a6591, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344191337 2024-12-04T20:29:56,059 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1d579f534c497290abaf3c4b76e1f0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733344192529 2024-12-04T20:29:56,059 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting d21ef4b39c8e4231a815ccc1a8c2c8b9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733344192529 2024-12-04T20:29:56,060 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting ea7ac08b11364f428821440c9654bd62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733344194777 2024-12-04T20:29:56,060 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80a9a8a5202f4be7a9f84313f7b0051c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733344194777 2024-12-04T20:29:56,077 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:56,077 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#B#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:56,077 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/17d56113b13843329195f52eda34d4ce is 50, key is test_row_0/B:col10/1733344194779/Put/seqid=0 2024-12-04T20:29:56,077 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8c76fc164fcc4897a7ba73c09dae84f6 is 50, key is test_row_0/A:col10/1733344194779/Put/seqid=0 2024-12-04T20:29:56,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742328_1504 (size=12561) 2024-12-04T20:29:56,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742327_1503 (size=12561) 2024-12-04T20:29:56,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T20:29:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:56,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T20:29:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/09caf407e88b4742ab6a9ccd45309e8d is 50, key is test_row_0/A:col10/1733344194804/Put/seqid=0 2024-12-04T20:29:56,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742329_1505 (size=9857) 2024-12-04T20:29:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T20:29:56,491 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/8c76fc164fcc4897a7ba73c09dae84f6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8c76fc164fcc4897a7ba73c09dae84f6 2024-12-04T20:29:56,494 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/17d56113b13843329195f52eda34d4ce as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/17d56113b13843329195f52eda34d4ce 2024-12-04T20:29:56,494 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/A of 4bd7564ec5d487ee4345848806acd5a1 into 8c76fc164fcc4897a7ba73c09dae84f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:56,494 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:56,494 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/A, priority=13, startTime=1733344196054; duration=0sec 2024-12-04T20:29:56,495 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:56,495 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:A 2024-12-04T20:29:56,495 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:56,495 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:56,495 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/C is initiating minor compaction (all files) 2024-12-04T20:29:56,495 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/C in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:56,495 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/ec34d583981e480f8f74e61829fab1f8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/7c23a07a92704d43981dc7f2fe39043f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9ef3d38967154b47a8e7e8b40bbcf543] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=33.6 K 2024-12-04T20:29:56,496 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec34d583981e480f8f74e61829fab1f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733344191337 2024-12-04T20:29:56,496 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c23a07a92704d43981dc7f2fe39043f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733344192529 2024-12-04T20:29:56,496 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef3d38967154b47a8e7e8b40bbcf543, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733344194777 2024-12-04T20:29:56,498 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/B of 4bd7564ec5d487ee4345848806acd5a1 into 17d56113b13843329195f52eda34d4ce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:56,498 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:56,498 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/B, priority=13, startTime=1733344196054; duration=0sec 2024-12-04T20:29:56,498 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:56,498 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:B 2024-12-04T20:29:56,501 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:56,501 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/c45e72c24f76449b84a91f3b49e882f5 is 50, key is test_row_0/C:col10/1733344194779/Put/seqid=0 2024-12-04T20:29:56,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742330_1506 (size=12561) 2024-12-04T20:29:56,593 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/09caf407e88b4742ab6a9ccd45309e8d 2024-12-04T20:29:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/3eafe2cbd4f54973b7d002f5f1cc73f4 is 50, key is test_row_0/B:col10/1733344194804/Put/seqid=0 2024-12-04T20:29:56,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742331_1507 (size=9857) 2024-12-04T20:29:56,907 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/c45e72c24f76449b84a91f3b49e882f5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c45e72c24f76449b84a91f3b49e882f5 2024-12-04T20:29:56,910 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/C of 4bd7564ec5d487ee4345848806acd5a1 into c45e72c24f76449b84a91f3b49e882f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:56,910 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:56,910 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/C, priority=13, startTime=1733344196054; duration=0sec 2024-12-04T20:29:56,910 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:56,910 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:C 2024-12-04T20:29:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:56,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:56,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733344196981, max timestamp=1733344196993Num uniques 859; 2024-12-04T20:29:56,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T20:29:56,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733344196981, max timestamp=1733344196993Num uniques 859; 2024-12-04T20:29:56,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T20:29:56,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733344196981, max timestamp=1733344196993Num uniques 859; 2024-12-04T20:29:56,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T20:29:57,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/3eafe2cbd4f54973b7d002f5f1cc73f4 2024-12-04T20:29:57,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/68c85bf09b67455aa2f52c5c282daba2 is 50, key is test_row_0/C:col10/1733344194804/Put/seqid=0 2024-12-04T20:29:57,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742332_1508 (size=9857) 2024-12-04T20:29:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344257030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344257030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344257031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344257142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344257142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344257142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344257352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344257352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344257352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T20:29:57,422 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/68c85bf09b67455aa2f52c5c282daba2 2024-12-04T20:29:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/09caf407e88b4742ab6a9ccd45309e8d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/09caf407e88b4742ab6a9ccd45309e8d 2024-12-04T20:29:57,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/09caf407e88b4742ab6a9ccd45309e8d, entries=100, sequenceid=262, filesize=9.6 K 2024-12-04T20:29:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/3eafe2cbd4f54973b7d002f5f1cc73f4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/3eafe2cbd4f54973b7d002f5f1cc73f4 2024-12-04T20:29:57,435 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/3eafe2cbd4f54973b7d002f5f1cc73f4, entries=100, sequenceid=262, filesize=9.6 K 2024-12-04T20:29:57,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/68c85bf09b67455aa2f52c5c282daba2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68c85bf09b67455aa2f52c5c282daba2 2024-12-04T20:29:57,439 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68c85bf09b67455aa2f52c5c282daba2, entries=100, sequenceid=262, filesize=9.6 K 2024-12-04T20:29:57,439 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 4bd7564ec5d487ee4345848806acd5a1 in 1276ms, sequenceid=262, compaction requested=false 2024-12-04T20:29:57,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:57,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:57,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-04T20:29:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-04T20:29:57,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-04T20:29:57,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1970 sec 2024-12-04T20:29:57,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.2070 sec 2024-12-04T20:29:57,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-04T20:29:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:57,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/b5fc731ed0424754b5d8bb8b27d265c3 is 50, key is test_row_0/A:col10/1733344197665/Put/seqid=0 2024-12-04T20:29:57,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742333_1509 (size=14741) 2024-12-04T20:29:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344257684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344257685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344257686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344257802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344257802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344257802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344258012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344258012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344258021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/b5fc731ed0424754b5d8bb8b27d265c3 2024-12-04T20:29:58,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/64cd81f1031c420b82bcce05a0f51966 is 50, key is test_row_0/B:col10/1733344197665/Put/seqid=0 2024-12-04T20:29:58,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742334_1510 (size=12301) 2024-12-04T20:29:58,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344258331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344258331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344258342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/64cd81f1031c420b82bcce05a0f51966 2024-12-04T20:29:58,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/f78744243d4f4aebb4ddb1970b446eb9 is 50, key is test_row_0/C:col10/1733344197665/Put/seqid=0 2024-12-04T20:29:58,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742335_1511 (size=12301) 2024-12-04T20:29:58,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/f78744243d4f4aebb4ddb1970b446eb9 2024-12-04T20:29:58,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/b5fc731ed0424754b5d8bb8b27d265c3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/b5fc731ed0424754b5d8bb8b27d265c3 2024-12-04T20:29:58,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/b5fc731ed0424754b5d8bb8b27d265c3, entries=200, sequenceid=302, filesize=14.4 K 2024-12-04T20:29:58,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/64cd81f1031c420b82bcce05a0f51966 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/64cd81f1031c420b82bcce05a0f51966 2024-12-04T20:29:58,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/64cd81f1031c420b82bcce05a0f51966, entries=150, sequenceid=302, filesize=12.0 K 2024-12-04T20:29:58,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/f78744243d4f4aebb4ddb1970b446eb9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f78744243d4f4aebb4ddb1970b446eb9 2024-12-04T20:29:58,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f78744243d4f4aebb4ddb1970b446eb9, entries=150, sequenceid=302, filesize=12.0 K 2024-12-04T20:29:58,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 4bd7564ec5d487ee4345848806acd5a1 in 856ms, sequenceid=302, compaction requested=true 2024-12-04T20:29:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:29:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:58,523 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:29:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:29:58,523 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:58,525 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:58,525 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37159 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:58,525 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/B is initiating minor compaction (all files) 2024-12-04T20:29:58,525 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/A is initiating minor compaction (all files) 2024-12-04T20:29:58,525 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/A in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:58,525 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/B in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:58,526 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/17d56113b13843329195f52eda34d4ce, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/3eafe2cbd4f54973b7d002f5f1cc73f4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/64cd81f1031c420b82bcce05a0f51966] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=33.9 K 2024-12-04T20:29:58,526 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8c76fc164fcc4897a7ba73c09dae84f6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/09caf407e88b4742ab6a9ccd45309e8d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/b5fc731ed0424754b5d8bb8b27d265c3] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=36.3 K 2024-12-04T20:29:58,526 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 17d56113b13843329195f52eda34d4ce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733344194777 2024-12-04T20:29:58,526 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c76fc164fcc4897a7ba73c09dae84f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733344194777 2024-12-04T20:29:58,526 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09caf407e88b4742ab6a9ccd45309e8d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733344194804 2024-12-04T20:29:58,526 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eafe2cbd4f54973b7d002f5f1cc73f4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733344194804 2024-12-04T20:29:58,526 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 64cd81f1031c420b82bcce05a0f51966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733344197011 2024-12-04T20:29:58,526 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5fc731ed0424754b5d8bb8b27d265c3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733344197011 2024-12-04T20:29:58,532 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:58,532 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:58,532 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/acd76e33085e457c95f0bbade2e53cd1 is 50, key is test_row_0/B:col10/1733344197665/Put/seqid=0 2024-12-04T20:29:58,533 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/3d12ddb64076416980b2a81f17553a3b is 50, key is test_row_0/A:col10/1733344197665/Put/seqid=0 2024-12-04T20:29:58,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742336_1512 (size=12813) 2024-12-04T20:29:58,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742337_1513 (size=12813) 2024-12-04T20:29:58,565 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/3d12ddb64076416980b2a81f17553a3b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/3d12ddb64076416980b2a81f17553a3b 2024-12-04T20:29:58,573 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/A of 4bd7564ec5d487ee4345848806acd5a1 into 3d12ddb64076416980b2a81f17553a3b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:58,573 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:58,573 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/A, priority=13, startTime=1733344198523; duration=0sec 2024-12-04T20:29:58,573 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:29:58,573 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:A 2024-12-04T20:29:58,573 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:29:58,574 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:29:58,574 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/C is initiating minor compaction (all files) 2024-12-04T20:29:58,574 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/C in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:58,574 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c45e72c24f76449b84a91f3b49e882f5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68c85bf09b67455aa2f52c5c282daba2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f78744243d4f4aebb4ddb1970b446eb9] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=33.9 K 2024-12-04T20:29:58,575 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45e72c24f76449b84a91f3b49e882f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733344194777 2024-12-04T20:29:58,575 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c85bf09b67455aa2f52c5c282daba2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733344194804 2024-12-04T20:29:58,575 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f78744243d4f4aebb4ddb1970b446eb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733344197011 2024-12-04T20:29:58,584 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:29:58,585 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/04fc05c26a824304a6ea3c90ed3379d3 is 50, key is test_row_0/C:col10/1733344197665/Put/seqid=0 2024-12-04T20:29:58,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742338_1514 (size=12813) 2024-12-04T20:29:58,619 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/04fc05c26a824304a6ea3c90ed3379d3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/04fc05c26a824304a6ea3c90ed3379d3 2024-12-04T20:29:58,625 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/C of 4bd7564ec5d487ee4345848806acd5a1 into 04fc05c26a824304a6ea3c90ed3379d3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:58,625 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:58,625 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/C, priority=13, startTime=1733344198523; duration=0sec 2024-12-04T20:29:58,625 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:58,625 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:C 2024-12-04T20:29:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:58,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:29:58,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:58,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:58,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:58,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:58,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:58,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:29:58,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/cbcbc602422d47e49c652f99f032afe1 is 50, key is test_row_0/A:col10/1733344197683/Put/seqid=0 2024-12-04T20:29:58,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742339_1515 (size=14741) 2024-12-04T20:29:58,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/cbcbc602422d47e49c652f99f032afe1 2024-12-04T20:29:58,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a712c99bb7b14a1a9cea4cd4f4295587 is 50, key is test_row_0/B:col10/1733344197683/Put/seqid=0 2024-12-04T20:29:58,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344198877, max timestamp=1733344198898Num uniques 859; 2024-12-04T20:29:58,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:58,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344198877, max timestamp=1733344198898Num uniques 859; 2024-12-04T20:29:58,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344198877, max timestamp=1733344198898Num uniques 859; 2024-12-04T20:29:58,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:58,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:29:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742340_1516 (size=12301) 2024-12-04T20:29:58,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344258952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344258954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,963 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/acd76e33085e457c95f0bbade2e53cd1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/acd76e33085e457c95f0bbade2e53cd1 2024-12-04T20:29:58,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344258955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:58,971 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/B of 4bd7564ec5d487ee4345848806acd5a1 into acd76e33085e457c95f0bbade2e53cd1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:29:58,971 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:58,972 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/B, priority=13, startTime=1733344198523; duration=0sec 2024-12-04T20:29:58,972 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:29:58,972 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:B 2024-12-04T20:29:59,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344259072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344259072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344259072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344259291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344259291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344259292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a712c99bb7b14a1a9cea4cd4f4295587 2024-12-04T20:29:59,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e078a25a9e4b4b879a2a1ea736815c4f is 50, key is test_row_0/C:col10/1733344197683/Put/seqid=0 2024-12-04T20:29:59,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742341_1517 (size=12301) 2024-12-04T20:29:59,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e078a25a9e4b4b879a2a1ea736815c4f 2024-12-04T20:29:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/cbcbc602422d47e49c652f99f032afe1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/cbcbc602422d47e49c652f99f032afe1 2024-12-04T20:29:59,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/cbcbc602422d47e49c652f99f032afe1, entries=200, sequenceid=314, filesize=14.4 K 2024-12-04T20:29:59,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a712c99bb7b14a1a9cea4cd4f4295587 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a712c99bb7b14a1a9cea4cd4f4295587 2024-12-04T20:29:59,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a712c99bb7b14a1a9cea4cd4f4295587, entries=150, sequenceid=314, filesize=12.0 K 2024-12-04T20:29:59,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e078a25a9e4b4b879a2a1ea736815c4f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e078a25a9e4b4b879a2a1ea736815c4f 2024-12-04T20:29:59,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e078a25a9e4b4b879a2a1ea736815c4f, entries=150, sequenceid=314, filesize=12.0 K 2024-12-04T20:29:59,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4bd7564ec5d487ee4345848806acd5a1 in 514ms, sequenceid=314, compaction requested=false 2024-12-04T20:29:59,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:29:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T20:29:59,402 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:29:59,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:29:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-04T20:29:59,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:29:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T20:29:59,407 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:29:59,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:29:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T20:29:59,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T20:29:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:29:59,560 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T20:29:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:29:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:29:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:29:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:29:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/f1c2b23cd8844ee5ac4d369e91e26664 is 50, key is test_row_0/A:col10/1733344198953/Put/seqid=0 2024-12-04T20:29:59,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742342_1518 (size=12301) 2024-12-04T20:29:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:29:59,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:29:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344259630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344259632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344259640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T20:29:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344259751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344259751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344259762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344259971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344259971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:29:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344259981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:29:59,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/f1c2b23cd8844ee5ac4d369e91e26664 2024-12-04T20:29:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:29:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:29:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/0b2625e60e904ac5b50444e7e9dc7c0d is 50, key is test_row_0/B:col10/1733344198953/Put/seqid=0 2024-12-04T20:30:00,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742343_1519 (size=12301) 2024-12-04T20:30:00,003 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/0b2625e60e904ac5b50444e7e9dc7c0d 2024-12-04T20:30:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/d5bc13bdb62c4f0da24d96281ec27c2b is 50, key is test_row_0/C:col10/1733344198953/Put/seqid=0 2024-12-04T20:30:00,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742344_1520 (size=12301) 2024-12-04T20:30:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T20:30:00,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344260281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344260291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344260291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:00,413 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/d5bc13bdb62c4f0da24d96281ec27c2b 2024-12-04T20:30:00,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/f1c2b23cd8844ee5ac4d369e91e26664 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c2b23cd8844ee5ac4d369e91e26664 2024-12-04T20:30:00,418 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c2b23cd8844ee5ac4d369e91e26664, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T20:30:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/0b2625e60e904ac5b50444e7e9dc7c0d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/0b2625e60e904ac5b50444e7e9dc7c0d 2024-12-04T20:30:00,421 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/0b2625e60e904ac5b50444e7e9dc7c0d, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T20:30:00,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/d5bc13bdb62c4f0da24d96281ec27c2b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/d5bc13bdb62c4f0da24d96281ec27c2b 2024-12-04T20:30:00,424 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/d5bc13bdb62c4f0da24d96281ec27c2b, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T20:30:00,425 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 4bd7564ec5d487ee4345848806acd5a1 in 865ms, sequenceid=350, compaction requested=true 2024-12-04T20:30:00,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:30:00,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:00,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-04T20:30:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-04T20:30:00,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-04T20:30:00,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-12-04T20:30:00,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0240 sec 2024-12-04T20:30:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T20:30:00,541 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:00,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-04T20:30:00,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T20:30:00,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:00,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T20:30:00,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T20:30:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:00,696 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:30:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:30:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:30:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:30:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/f1c65842b3c9412b915119e21964f557 is 50, key is test_row_0/A:col10/1733344199629/Put/seqid=0 2024-12-04T20:30:00,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742345_1521 (size=12301) 2024-12-04T20:30:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:30:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. as already flushing 2024-12-04T20:30:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T20:30:00,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344200889, max timestamp=1733344200894Num uniques 859; 2024-12-04T20:30:00,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:30:00,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344200889, max timestamp=1733344200894Num uniques 859; 2024-12-04T20:30:00,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:30:00,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344200889, max timestamp=1733344200894Num uniques 859; 2024-12-04T20:30:00,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:30:00,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344260935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:00,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344260939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:00,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344260942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344261051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344261052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344261052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,115 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/f1c65842b3c9412b915119e21964f557 2024-12-04T20:30:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a48b32fc821a4861b26bad535ef1c2a3 is 50, key is test_row_0/B:col10/1733344199629/Put/seqid=0 2024-12-04T20:30:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742346_1522 (size=12301) 2024-12-04T20:30:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T20:30:01,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344261261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344261262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344261262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,523 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a48b32fc821a4861b26bad535ef1c2a3 2024-12-04T20:30:01,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e93096e48bce4c8d985eaefd218fcb37 is 50, key is test_row_0/C:col10/1733344199629/Put/seqid=0 2024-12-04T20:30:01,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742347_1523 (size=12301) 2024-12-04T20:30:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344261571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344261571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344261571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T20:30:01,931 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e93096e48bce4c8d985eaefd218fcb37 2024-12-04T20:30:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/f1c65842b3c9412b915119e21964f557 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c65842b3c9412b915119e21964f557 2024-12-04T20:30:01,938 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c65842b3c9412b915119e21964f557, entries=150, sequenceid=360, filesize=12.0 K 2024-12-04T20:30:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/a48b32fc821a4861b26bad535ef1c2a3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a48b32fc821a4861b26bad535ef1c2a3 2024-12-04T20:30:01,942 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a48b32fc821a4861b26bad535ef1c2a3, entries=150, sequenceid=360, filesize=12.0 K 2024-12-04T20:30:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/e93096e48bce4c8d985eaefd218fcb37 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e93096e48bce4c8d985eaefd218fcb37 2024-12-04T20:30:01,946 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e93096e48bce4c8d985eaefd218fcb37, entries=150, sequenceid=360, filesize=12.0 K 2024-12-04T20:30:01,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4bd7564ec5d487ee4345848806acd5a1 in 1251ms, sequenceid=360, compaction requested=true 2024-12-04T20:30:01,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:30:01,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:01,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-04T20:30:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-04T20:30:01,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-04T20:30:01,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-12-04T20:30:01,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4090 sec 2024-12-04T20:30:02,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344200932, max timestamp=1733344202096Num uniques 1018; 2024-12-04T20:30:02,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344200932, max timestamp=1733344202096Num uniques 1018; 2024-12-04T20:30:02,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:30:02,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733344200932, max timestamp=1733344202096Num uniques 1018; 2024-12-04T20:30:02,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:30:02,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T20:30:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:30:02,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-04T20:30:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:30:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:30:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:30:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:02,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/1150327046f444368604b0a8407a9dcf is 50, key is test_row_0/A:col10/1733344202107/Put/seqid=0 2024-12-04T20:30:02,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742348_1524 (size=14741) 2024-12-04T20:30:02,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/1150327046f444368604b0a8407a9dcf 2024-12-04T20:30:02,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/4c9341b86fa34cbc8a63e093ce99a885 is 50, key is test_row_0/B:col10/1733344202107/Put/seqid=0 2024-12-04T20:30:02,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742349_1525 (size=12301) 2024-12-04T20:30:02,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/4c9341b86fa34cbc8a63e093ce99a885 2024-12-04T20:30:02,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/10496574d30242588ed47260ee6139e7 is 50, key is test_row_0/C:col10/1733344202107/Put/seqid=0 2024-12-04T20:30:02,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742350_1526 (size=12301) 2024-12-04T20:30:02,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344202151, max timestamp=1733344202164Num uniques 859; 2024-12-04T20:30:02,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344202151, max timestamp=1733344202164Num uniques 859; 2024-12-04T20:30:02,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733344202151, max timestamp=1733344202164Num uniques 859; 2024-12-04T20:30:02,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:30:02,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:30:02,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:30:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344262207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344262208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344262212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:02,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:02,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54426 deadline: 1733344262324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54406 deadline: 1733344262324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:02,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54408 deadline: 1733344262324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 , the old value is region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., hostname=c2ef38372881,45443,1733344054291, seqNum=2 because the exception is null or not the one we care about 2024-12-04T20:30:02,485 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-12-04T20:30:02,485 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,485 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-12-04T20:30:02,485 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,485 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,485 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,485 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,485 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,486 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-12-04T20:30:02,486 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,486 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,486 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,495 INFO [Thread-2102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2102. 2024-12-04T20:30:02,495 DEBUG [Thread-2102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,495 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,495 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,496 INFO [Thread-2100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2100. 2024-12-04T20:30:02,496 DEBUG [Thread-2100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,496 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,496 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/10496574d30242588ed47260ee6139e7 2024-12-04T20:30:02,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/1150327046f444368604b0a8407a9dcf as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1150327046f444368604b0a8407a9dcf 2024-12-04T20:30:02,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1150327046f444368604b0a8407a9dcf, entries=200, sequenceid=403, filesize=14.4 K 2024-12-04T20:30:02,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/4c9341b86fa34cbc8a63e093ce99a885 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/4c9341b86fa34cbc8a63e093ce99a885 2024-12-04T20:30:02,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/4c9341b86fa34cbc8a63e093ce99a885, entries=150, sequenceid=403, filesize=12.0 K 2024-12-04T20:30:02,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/10496574d30242588ed47260ee6139e7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/10496574d30242588ed47260ee6139e7 2024-12-04T20:30:02,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/10496574d30242588ed47260ee6139e7, entries=150, sequenceid=403, filesize=12.0 K 2024-12-04T20:30:02,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.95 KB/168912, currentSize=127.47 KB/130530 for 4bd7564ec5d487ee4345848806acd5a1 in 441ms, sequenceid=403, compaction requested=true 2024-12-04T20:30:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:30:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:30:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:30:02,549 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T20:30:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:02,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4bd7564ec5d487ee4345848806acd5a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:30:02,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:02,550 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 66897 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62017 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/A is initiating minor compaction (all files) 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/B is initiating minor compaction (all files) 2024-12-04T20:30:02,551 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/A in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:02,551 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/B in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:02,551 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/3d12ddb64076416980b2a81f17553a3b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/cbcbc602422d47e49c652f99f032afe1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c2b23cd8844ee5ac4d369e91e26664, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c65842b3c9412b915119e21964f557, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1150327046f444368604b0a8407a9dcf] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=65.3 K 2024-12-04T20:30:02,551 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/acd76e33085e457c95f0bbade2e53cd1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a712c99bb7b14a1a9cea4cd4f4295587, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/0b2625e60e904ac5b50444e7e9dc7c0d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a48b32fc821a4861b26bad535ef1c2a3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/4c9341b86fa34cbc8a63e093ce99a885] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=60.6 K 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting acd76e33085e457c95f0bbade2e53cd1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733344197011 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d12ddb64076416980b2a81f17553a3b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733344197011 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbcbc602422d47e49c652f99f032afe1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733344197681 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a712c99bb7b14a1a9cea4cd4f4295587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733344197681 2024-12-04T20:30:02,551 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1c2b23cd8844ee5ac4d369e91e26664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733344198923 2024-12-04T20:30:02,552 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2625e60e904ac5b50444e7e9dc7c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733344198923 2024-12-04T20:30:02,552 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1c65842b3c9412b915119e21964f557, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733344199627 2024-12-04T20:30:02,552 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting a48b32fc821a4861b26bad535ef1c2a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733344199627 2024-12-04T20:30:02,552 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9341b86fa34cbc8a63e093ce99a885, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733344200932 2024-12-04T20:30:02,552 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1150327046f444368604b0a8407a9dcf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733344200932 2024-12-04T20:30:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:30:02,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-04T20:30:02,553 INFO [Thread-2087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2087. 2024-12-04T20:30:02,553 INFO [Thread-2089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2089. 2024-12-04T20:30:02,553 INFO [Thread-2091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2091. 2024-12-04T20:30:02,553 DEBUG [Thread-2087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,553 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,553 DEBUG [Thread-2091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,553 DEBUG [Thread-2089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:02,553 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,553 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:30:02,553 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,553 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,553 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:02,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:02,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:30:02,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:02,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:30:02,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:02,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:02,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/04fab9e2a7764f13999e4ea137705e72 is 50, key is test_row_0/A:col10/1733344202552/Put/seqid=0 2024-12-04T20:30:02,566 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#A#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:02,566 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#B#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:02,566 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/44324464441a48e28696b5c2a5161a42 is 50, key is test_row_0/A:col10/1733344202107/Put/seqid=0 2024-12-04T20:30:02,567 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/6e0b13677e9042418c6ffee78f1d8be8 is 50, key is test_row_0/B:col10/1733344202107/Put/seqid=0 2024-12-04T20:30:02,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742351_1527 (size=12301) 2024-12-04T20:30:02,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742352_1528 (size=12983) 2024-12-04T20:30:02,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742353_1529 (size=12983) 2024-12-04T20:30:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T20:30:02,691 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:02,787 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:30:02,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/04fab9e2a7764f13999e4ea137705e72 2024-12-04T20:30:02,972 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/44324464441a48e28696b5c2a5161a42 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/44324464441a48e28696b5c2a5161a42 2024-12-04T20:30:02,972 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/6e0b13677e9042418c6ffee78f1d8be8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6e0b13677e9042418c6ffee78f1d8be8 2024-12-04T20:30:02,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/e2320c5e8c274d4794895b561948f558 is 50, key is test_row_0/B:col10/1733344202552/Put/seqid=0 2024-12-04T20:30:02,976 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/B of 4bd7564ec5d487ee4345848806acd5a1 into 6e0b13677e9042418c6ffee78f1d8be8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:02,976 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/A of 4bd7564ec5d487ee4345848806acd5a1 into 44324464441a48e28696b5c2a5161a42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:02,976 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:30:02,976 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:30:02,976 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/B, priority=11, startTime=1733344202549; duration=0sec 2024-12-04T20:30:02,976 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/A, priority=11, startTime=1733344202549; duration=0sec 2024-12-04T20:30:02,976 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:02,976 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:B 2024-12-04T20:30:02,976 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:02,976 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:A 2024-12-04T20:30:02,976 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T20:30:02,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742354_1530 (size=12301) 2024-12-04T20:30:02,977 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62017 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T20:30:02,977 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): 4bd7564ec5d487ee4345848806acd5a1/C is initiating minor compaction (all files) 2024-12-04T20:30:02,977 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4bd7564ec5d487ee4345848806acd5a1/C in TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:02,977 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/04fc05c26a824304a6ea3c90ed3379d3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e078a25a9e4b4b879a2a1ea736815c4f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/d5bc13bdb62c4f0da24d96281ec27c2b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e93096e48bce4c8d985eaefd218fcb37, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/10496574d30242588ed47260ee6139e7] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp, totalSize=60.6 K 2024-12-04T20:30:02,978 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fc05c26a824304a6ea3c90ed3379d3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733344197011 2024-12-04T20:30:02,978 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e078a25a9e4b4b879a2a1ea736815c4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733344197681 2024-12-04T20:30:02,978 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d5bc13bdb62c4f0da24d96281ec27c2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733344198923 2024-12-04T20:30:02,978 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e93096e48bce4c8d985eaefd218fcb37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733344199627 2024-12-04T20:30:02,978 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 10496574d30242588ed47260ee6139e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733344200932 2024-12-04T20:30:02,984 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4bd7564ec5d487ee4345848806acd5a1#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:02,984 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/35c420a4239849d599330cde9cd842d5 is 50, key is test_row_0/C:col10/1733344202107/Put/seqid=0 2024-12-04T20:30:02,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742355_1531 (size=12983) 2024-12-04T20:30:03,223 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-12-04T20:30:03,223 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:03,223 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:03,223 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:03,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:03,273 INFO [Thread-2083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2083. 2024-12-04T20:30:03,273 DEBUG [Thread-2083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:03,273 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:03,273 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:03,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1585 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4754 rows 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1587 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4761 rows 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1594 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4782 rows 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1583 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4749 rows 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1584 2024-12-04T20:30:03,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4752 rows 2024-12-04T20:30:03,273 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:30:03,273 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:30:03,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:03,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:03,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:03,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T20:30:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T20:30:03,276 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344203276"}]},"ts":"1733344203276"} 2024-12-04T20:30:03,277 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T20:30:03,278 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T20:30:03,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:30:03,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, UNASSIGN}] 2024-12-04T20:30:03,279 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, UNASSIGN 2024-12-04T20:30:03,280 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=4bd7564ec5d487ee4345848806acd5a1, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:30:03,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, UNASSIGN because future has completed 2024-12-04T20:30:03,281 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:30:03,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:30:03,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/e2320c5e8c274d4794895b561948f558 2024-12-04T20:30:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T20:30:03,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/58d51ddc04ca459f8afdaf44fa7b2a46 is 50, key is test_row_0/C:col10/1733344202552/Put/seqid=0 2024-12-04T20:30:03,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742356_1532 (size=12301) 2024-12-04T20:30:03,390 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/35c420a4239849d599330cde9cd842d5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/35c420a4239849d599330cde9cd842d5 2024-12-04T20:30:03,393 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4bd7564ec5d487ee4345848806acd5a1/C of 4bd7564ec5d487ee4345848806acd5a1 into 35c420a4239849d599330cde9cd842d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:03,393 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:30:03,394 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., storeName=4bd7564ec5d487ee4345848806acd5a1/C, priority=11, startTime=1733344202550; duration=0sec 2024-12-04T20:30:03,394 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:03,394 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4bd7564ec5d487ee4345848806acd5a1:C 2024-12-04T20:30:03,433 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:30:03,433 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:30:03,433 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 4bd7564ec5d487ee4345848806acd5a1, disabling compactions & flushes 2024-12-04T20:30:03,433 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T20:30:03,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/58d51ddc04ca459f8afdaf44fa7b2a46 2024-12-04T20:30:03,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/04fab9e2a7764f13999e4ea137705e72 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/04fab9e2a7764f13999e4ea137705e72 2024-12-04T20:30:03,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/04fab9e2a7764f13999e4ea137705e72, entries=150, sequenceid=440, filesize=12.0 K 2024-12-04T20:30:03,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/e2320c5e8c274d4794895b561948f558 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/e2320c5e8c274d4794895b561948f558 2024-12-04T20:30:03,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/e2320c5e8c274d4794895b561948f558, entries=150, sequenceid=440, filesize=12.0 K 2024-12-04T20:30:03,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/58d51ddc04ca459f8afdaf44fa7b2a46 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/58d51ddc04ca459f8afdaf44fa7b2a46 2024-12-04T20:30:03,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/58d51ddc04ca459f8afdaf44fa7b2a46, entries=150, sequenceid=440, filesize=12.0 K 2024-12-04T20:30:03,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=13.42 KB/13740 for 4bd7564ec5d487ee4345848806acd5a1 in 1249ms, sequenceid=440, compaction requested=false 2024-12-04T20:30:03,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4bd7564ec5d487ee4345848806acd5a1: 2024-12-04T20:30:03,802 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. after waiting 0 ms 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:03,802 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 4bd7564ec5d487ee4345848806acd5a1 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=A 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=B 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:03,802 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4bd7564ec5d487ee4345848806acd5a1, store=C 2024-12-04T20:30:03,803 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:03,806 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/21d5e76ac2f84a75b9be81a57ca07b5a is 50, key is test_row_0/A:col10/1733344203221/Put/seqid=0 2024-12-04T20:30:03,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742357_1533 (size=9857) 2024-12-04T20:30:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T20:30:04,210 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/21d5e76ac2f84a75b9be81a57ca07b5a 2024-12-04T20:30:04,215 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/af4cb334f5ac4c658d04925f06336478 is 50, key is test_row_0/B:col10/1733344203221/Put/seqid=0 2024-12-04T20:30:04,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742358_1534 (size=9857) 2024-12-04T20:30:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T20:30:04,618 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/af4cb334f5ac4c658d04925f06336478 2024-12-04T20:30:04,624 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/bfec4142668149d6b5cc99126838796d is 50, key is test_row_0/C:col10/1733344203221/Put/seqid=0 2024-12-04T20:30:04,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742359_1535 (size=9857) 2024-12-04T20:30:05,030 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/bfec4142668149d6b5cc99126838796d 2024-12-04T20:30:05,033 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/A/21d5e76ac2f84a75b9be81a57ca07b5a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/21d5e76ac2f84a75b9be81a57ca07b5a 2024-12-04T20:30:05,036 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/21d5e76ac2f84a75b9be81a57ca07b5a, entries=100, sequenceid=448, filesize=9.6 K 2024-12-04T20:30:05,036 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/B/af4cb334f5ac4c658d04925f06336478 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/af4cb334f5ac4c658d04925f06336478 2024-12-04T20:30:05,039 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/af4cb334f5ac4c658d04925f06336478, entries=100, sequenceid=448, filesize=9.6 K 2024-12-04T20:30:05,039 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/.tmp/C/bfec4142668149d6b5cc99126838796d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/bfec4142668149d6b5cc99126838796d 2024-12-04T20:30:05,042 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/bfec4142668149d6b5cc99126838796d, entries=100, sequenceid=448, filesize=9.6 K 2024-12-04T20:30:05,042 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 4bd7564ec5d487ee4345848806acd5a1 in 1240ms, sequenceid=448, compaction requested=true 2024-12-04T20:30:05,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8b2d9d929f284439b334f895a2febf25, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/629fa9dd6fcd4c2d98a3b72834513cb2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/62241ff5807f41b7a20f01f0461bcda7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/00c2187bf83040b6b127b85924486c4b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/74ef5019b5fc4bdf95e923a4638ef93b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d5a40554ee184562bf8385aaf5ae091e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1e01669b2a09444dab1b2967a41374a9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8af2677fa2704499bc6a3e419aa8014c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/ad2422a9ab314978b64dcefb08c55d52, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/605f5219bb64409db8919470308a6591, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d21ef4b39c8e4231a815ccc1a8c2c8b9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/80a9a8a5202f4be7a9f84313f7b0051c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8c76fc164fcc4897a7ba73c09dae84f6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/09caf407e88b4742ab6a9ccd45309e8d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/b5fc731ed0424754b5d8bb8b27d265c3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/3d12ddb64076416980b2a81f17553a3b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/cbcbc602422d47e49c652f99f032afe1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c2b23cd8844ee5ac4d369e91e26664, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c65842b3c9412b915119e21964f557, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1150327046f444368604b0a8407a9dcf] to archive 2024-12-04T20:30:05,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:30:05,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8b2d9d929f284439b334f895a2febf25 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8b2d9d929f284439b334f895a2febf25 2024-12-04T20:30:05,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/629fa9dd6fcd4c2d98a3b72834513cb2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/629fa9dd6fcd4c2d98a3b72834513cb2 2024-12-04T20:30:05,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/62241ff5807f41b7a20f01f0461bcda7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/62241ff5807f41b7a20f01f0461bcda7 2024-12-04T20:30:05,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/00c2187bf83040b6b127b85924486c4b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/00c2187bf83040b6b127b85924486c4b 2024-12-04T20:30:05,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/74ef5019b5fc4bdf95e923a4638ef93b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/74ef5019b5fc4bdf95e923a4638ef93b 2024-12-04T20:30:05,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d5a40554ee184562bf8385aaf5ae091e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d5a40554ee184562bf8385aaf5ae091e 2024-12-04T20:30:05,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1e01669b2a09444dab1b2967a41374a9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1e01669b2a09444dab1b2967a41374a9 2024-12-04T20:30:05,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8af2677fa2704499bc6a3e419aa8014c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8af2677fa2704499bc6a3e419aa8014c 2024-12-04T20:30:05,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/ad2422a9ab314978b64dcefb08c55d52 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/ad2422a9ab314978b64dcefb08c55d52 2024-12-04T20:30:05,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/605f5219bb64409db8919470308a6591 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/605f5219bb64409db8919470308a6591 2024-12-04T20:30:05,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d21ef4b39c8e4231a815ccc1a8c2c8b9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/d21ef4b39c8e4231a815ccc1a8c2c8b9 2024-12-04T20:30:05,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/80a9a8a5202f4be7a9f84313f7b0051c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/80a9a8a5202f4be7a9f84313f7b0051c 2024-12-04T20:30:05,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8c76fc164fcc4897a7ba73c09dae84f6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/8c76fc164fcc4897a7ba73c09dae84f6 2024-12-04T20:30:05,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/09caf407e88b4742ab6a9ccd45309e8d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/09caf407e88b4742ab6a9ccd45309e8d 2024-12-04T20:30:05,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/b5fc731ed0424754b5d8bb8b27d265c3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/b5fc731ed0424754b5d8bb8b27d265c3 2024-12-04T20:30:05,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/3d12ddb64076416980b2a81f17553a3b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/3d12ddb64076416980b2a81f17553a3b 2024-12-04T20:30:05,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/cbcbc602422d47e49c652f99f032afe1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/cbcbc602422d47e49c652f99f032afe1 2024-12-04T20:30:05,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c2b23cd8844ee5ac4d369e91e26664 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c2b23cd8844ee5ac4d369e91e26664 2024-12-04T20:30:05,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c65842b3c9412b915119e21964f557 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/f1c65842b3c9412b915119e21964f557 2024-12-04T20:30:05,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1150327046f444368604b0a8407a9dcf to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/1150327046f444368604b0a8407a9dcf 2024-12-04T20:30:05,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/f772a078ae704c3e95945d3ac02eb62e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/12a00e112f6440a8aa3e19d6084bc9bc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/eb4a27bb282e45df87c982f74c38db55, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/5822389f4f2c489da379f986c1a4dea0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/9ae25021ea324d299e23f0bfc4b01c0f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6a2baf291c454e72a382f0f57ed3612f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a05ccf2e193e42228c023b3cfeaae633, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d22e52b11fd844daba39263fbd1e6d52, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/826ff7027a0d49adb9ecfbbdd0f149a8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d48966c17e064304a76032ddfdde1d14, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/bf1d579f534c497290abaf3c4b76e1f0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/17d56113b13843329195f52eda34d4ce, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/ea7ac08b11364f428821440c9654bd62, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/3eafe2cbd4f54973b7d002f5f1cc73f4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/acd76e33085e457c95f0bbade2e53cd1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/64cd81f1031c420b82bcce05a0f51966, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a712c99bb7b14a1a9cea4cd4f4295587, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/0b2625e60e904ac5b50444e7e9dc7c0d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a48b32fc821a4861b26bad535ef1c2a3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/4c9341b86fa34cbc8a63e093ce99a885] to archive 2024-12-04T20:30:05,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:30:05,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/f772a078ae704c3e95945d3ac02eb62e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/f772a078ae704c3e95945d3ac02eb62e 2024-12-04T20:30:05,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/12a00e112f6440a8aa3e19d6084bc9bc to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/12a00e112f6440a8aa3e19d6084bc9bc 2024-12-04T20:30:05,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/eb4a27bb282e45df87c982f74c38db55 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/eb4a27bb282e45df87c982f74c38db55 2024-12-04T20:30:05,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/5822389f4f2c489da379f986c1a4dea0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/5822389f4f2c489da379f986c1a4dea0 2024-12-04T20:30:05,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/9ae25021ea324d299e23f0bfc4b01c0f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/9ae25021ea324d299e23f0bfc4b01c0f 2024-12-04T20:30:05,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6a2baf291c454e72a382f0f57ed3612f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6a2baf291c454e72a382f0f57ed3612f 2024-12-04T20:30:05,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a05ccf2e193e42228c023b3cfeaae633 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a05ccf2e193e42228c023b3cfeaae633 2024-12-04T20:30:05,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d22e52b11fd844daba39263fbd1e6d52 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d22e52b11fd844daba39263fbd1e6d52 2024-12-04T20:30:05,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/826ff7027a0d49adb9ecfbbdd0f149a8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/826ff7027a0d49adb9ecfbbdd0f149a8 2024-12-04T20:30:05,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d48966c17e064304a76032ddfdde1d14 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/d48966c17e064304a76032ddfdde1d14 2024-12-04T20:30:05,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/bf1d579f534c497290abaf3c4b76e1f0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/bf1d579f534c497290abaf3c4b76e1f0 2024-12-04T20:30:05,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/17d56113b13843329195f52eda34d4ce to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/17d56113b13843329195f52eda34d4ce 2024-12-04T20:30:05,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/ea7ac08b11364f428821440c9654bd62 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/ea7ac08b11364f428821440c9654bd62 2024-12-04T20:30:05,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/3eafe2cbd4f54973b7d002f5f1cc73f4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/3eafe2cbd4f54973b7d002f5f1cc73f4 2024-12-04T20:30:05,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/acd76e33085e457c95f0bbade2e53cd1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/acd76e33085e457c95f0bbade2e53cd1 2024-12-04T20:30:05,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/64cd81f1031c420b82bcce05a0f51966 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/64cd81f1031c420b82bcce05a0f51966 2024-12-04T20:30:05,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a712c99bb7b14a1a9cea4cd4f4295587 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a712c99bb7b14a1a9cea4cd4f4295587 2024-12-04T20:30:05,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/0b2625e60e904ac5b50444e7e9dc7c0d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/0b2625e60e904ac5b50444e7e9dc7c0d 2024-12-04T20:30:05,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a48b32fc821a4861b26bad535ef1c2a3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/a48b32fc821a4861b26bad535ef1c2a3 2024-12-04T20:30:05,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/4c9341b86fa34cbc8a63e093ce99a885 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/4c9341b86fa34cbc8a63e093ce99a885 2024-12-04T20:30:05,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/fc90c082257d4c6d9a9d381e2efd34ea, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f2139123b75243bc8b697e72b52458a8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/717c2a4d1c744dcb952d2f59b40f8ab3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/0b48f9a1d7cb431596bb265eaa178743, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/89a1eb30e7a6438da09da98a5936e138, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9fe557fd6a2a4deab5e744b850f0b74b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e6b68f98b91f47c1b6d0204e3d95e6bb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c7aa5b38ea774115a4180420c9612ba3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/ec34d583981e480f8f74e61829fab1f8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68604435b32749f9a1561d8c075a6afc, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/7c23a07a92704d43981dc7f2fe39043f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c45e72c24f76449b84a91f3b49e882f5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9ef3d38967154b47a8e7e8b40bbcf543, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68c85bf09b67455aa2f52c5c282daba2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/04fc05c26a824304a6ea3c90ed3379d3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f78744243d4f4aebb4ddb1970b446eb9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e078a25a9e4b4b879a2a1ea736815c4f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/d5bc13bdb62c4f0da24d96281ec27c2b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e93096e48bce4c8d985eaefd218fcb37, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/10496574d30242588ed47260ee6139e7] to archive 2024-12-04T20:30:05,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:30:05,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/fc90c082257d4c6d9a9d381e2efd34ea to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/fc90c082257d4c6d9a9d381e2efd34ea 2024-12-04T20:30:05,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f2139123b75243bc8b697e72b52458a8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f2139123b75243bc8b697e72b52458a8 2024-12-04T20:30:05,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/717c2a4d1c744dcb952d2f59b40f8ab3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/717c2a4d1c744dcb952d2f59b40f8ab3 2024-12-04T20:30:05,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/0b48f9a1d7cb431596bb265eaa178743 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/0b48f9a1d7cb431596bb265eaa178743 2024-12-04T20:30:05,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/89a1eb30e7a6438da09da98a5936e138 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/89a1eb30e7a6438da09da98a5936e138 2024-12-04T20:30:05,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9fe557fd6a2a4deab5e744b850f0b74b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9fe557fd6a2a4deab5e744b850f0b74b 2024-12-04T20:30:05,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e6b68f98b91f47c1b6d0204e3d95e6bb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e6b68f98b91f47c1b6d0204e3d95e6bb 2024-12-04T20:30:05,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c7aa5b38ea774115a4180420c9612ba3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c7aa5b38ea774115a4180420c9612ba3 2024-12-04T20:30:05,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/ec34d583981e480f8f74e61829fab1f8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/ec34d583981e480f8f74e61829fab1f8 2024-12-04T20:30:05,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68604435b32749f9a1561d8c075a6afc to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68604435b32749f9a1561d8c075a6afc 2024-12-04T20:30:05,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/7c23a07a92704d43981dc7f2fe39043f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/7c23a07a92704d43981dc7f2fe39043f 2024-12-04T20:30:05,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c45e72c24f76449b84a91f3b49e882f5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/c45e72c24f76449b84a91f3b49e882f5 2024-12-04T20:30:05,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9ef3d38967154b47a8e7e8b40bbcf543 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/9ef3d38967154b47a8e7e8b40bbcf543 2024-12-04T20:30:05,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68c85bf09b67455aa2f52c5c282daba2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/68c85bf09b67455aa2f52c5c282daba2 2024-12-04T20:30:05,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/04fc05c26a824304a6ea3c90ed3379d3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/04fc05c26a824304a6ea3c90ed3379d3 2024-12-04T20:30:05,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f78744243d4f4aebb4ddb1970b446eb9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/f78744243d4f4aebb4ddb1970b446eb9 2024-12-04T20:30:05,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e078a25a9e4b4b879a2a1ea736815c4f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e078a25a9e4b4b879a2a1ea736815c4f 2024-12-04T20:30:05,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/d5bc13bdb62c4f0da24d96281ec27c2b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/d5bc13bdb62c4f0da24d96281ec27c2b 2024-12-04T20:30:05,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e93096e48bce4c8d985eaefd218fcb37 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/e93096e48bce4c8d985eaefd218fcb37 2024-12-04T20:30:05,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/10496574d30242588ed47260ee6139e7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/10496574d30242588ed47260ee6139e7 2024-12-04T20:30:05,087 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/recovered.edits/451.seqid, newMaxSeqId=451, maxSeqId=1 2024-12-04T20:30:05,087 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. 2024-12-04T20:30:05,087 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 4bd7564ec5d487ee4345848806acd5a1: Waiting for close lock at 1733344203433Running coprocessor pre-close hooks at 1733344203433Disabling compacts and flushes for region at 1733344203433Disabling writes for close at 1733344203802 (+369 ms)Obtaining lock to block concurrent updates at 1733344203802Preparing flush snapshotting stores in 4bd7564ec5d487ee4345848806acd5a1 at 1733344203802Finished memstore snapshotting TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733344203803 (+1 ms)Flushing stores of TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1. at 1733344203803Flushing 4bd7564ec5d487ee4345848806acd5a1/A: creating writer at 1733344203803Flushing 4bd7564ec5d487ee4345848806acd5a1/A: appending metadata at 1733344203805 (+2 ms)Flushing 4bd7564ec5d487ee4345848806acd5a1/A: closing flushed file at 1733344203805Flushing 4bd7564ec5d487ee4345848806acd5a1/B: creating writer at 1733344204213 (+408 ms)Flushing 4bd7564ec5d487ee4345848806acd5a1/B: appending metadata at 1733344204214 (+1 ms)Flushing 4bd7564ec5d487ee4345848806acd5a1/B: closing flushed file at 1733344204215 (+1 ms)Flushing 4bd7564ec5d487ee4345848806acd5a1/C: creating writer at 1733344204621 (+406 ms)Flushing 4bd7564ec5d487ee4345848806acd5a1/C: appending metadata at 1733344204623 (+2 ms)Flushing 4bd7564ec5d487ee4345848806acd5a1/C: closing flushed file at 1733344204623Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16bc766c: reopening flushed file at 1733344205033 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d5a95a7: reopening flushed file at 1733344205036 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6698f2: reopening flushed file at 1733344205039 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 4bd7564ec5d487ee4345848806acd5a1 in 1240ms, sequenceid=448, compaction requested=true at 1733344205042 (+3 ms)Writing region close event to WAL at 1733344205085 (+43 ms)Running coprocessor post-close hooks at 1733344205087 (+2 ms)Closed at 1733344205087 2024-12-04T20:30:05,089 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:30:05,089 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=4bd7564ec5d487ee4345848806acd5a1, regionState=CLOSED 2024-12-04T20:30:05,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:30:05,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-04T20:30:05,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 4bd7564ec5d487ee4345848806acd5a1, server=c2ef38372881,45443,1733344054291 in 1.8100 sec 2024-12-04T20:30:05,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-12-04T20:30:05,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4bd7564ec5d487ee4345848806acd5a1, UNASSIGN in 1.8130 sec 2024-12-04T20:30:05,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-04T20:30:05,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8160 sec 2024-12-04T20:30:05,096 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344205095"}]},"ts":"1733344205095"} 2024-12-04T20:30:05,096 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T20:30:05,096 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T20:30:05,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8230 sec 2024-12-04T20:30:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T20:30:05,421 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:05,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T20:30:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:05,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T20:30:05,424 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:05,425 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:30:05,427 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/recovered.edits] 2024-12-04T20:30:05,428 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/04fab9e2a7764f13999e4ea137705e72 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/04fab9e2a7764f13999e4ea137705e72 2024-12-04T20:30:05,429 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/21d5e76ac2f84a75b9be81a57ca07b5a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/21d5e76ac2f84a75b9be81a57ca07b5a 2024-12-04T20:30:05,430 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/44324464441a48e28696b5c2a5161a42 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/A/44324464441a48e28696b5c2a5161a42 2024-12-04T20:30:05,432 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6e0b13677e9042418c6ffee78f1d8be8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/6e0b13677e9042418c6ffee78f1d8be8 2024-12-04T20:30:05,433 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/af4cb334f5ac4c658d04925f06336478 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/af4cb334f5ac4c658d04925f06336478 2024-12-04T20:30:05,434 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/e2320c5e8c274d4794895b561948f558 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/B/e2320c5e8c274d4794895b561948f558 2024-12-04T20:30:05,435 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/35c420a4239849d599330cde9cd842d5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/35c420a4239849d599330cde9cd842d5 2024-12-04T20:30:05,436 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/58d51ddc04ca459f8afdaf44fa7b2a46 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/58d51ddc04ca459f8afdaf44fa7b2a46 2024-12-04T20:30:05,436 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/bfec4142668149d6b5cc99126838796d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/C/bfec4142668149d6b5cc99126838796d 2024-12-04T20:30:05,438 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/recovered.edits/451.seqid to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1/recovered.edits/451.seqid 2024-12-04T20:30:05,438 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/4bd7564ec5d487ee4345848806acd5a1 2024-12-04T20:30:05,438 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T20:30:05,440 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:05,441 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T20:30:05,443 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T20:30:05,443 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:05,444 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T20:30:05,444 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733344205444"}]},"ts":"9223372036854775807"} 2024-12-04T20:30:05,445 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T20:30:05,445 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4bd7564ec5d487ee4345848806acd5a1, NAME => 'TestAcidGuarantees,,1733344180266.4bd7564ec5d487ee4345848806acd5a1.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T20:30:05,445 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T20:30:05,445 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733344205445"}]},"ts":"9223372036854775807"} 2024-12-04T20:30:05,446 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T20:30:05,447 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:05,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-12-04T20:30:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T20:30:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T20:30:05,534 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:05,540 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 248) - Thread LEAK? -, OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 503), ProcessCount=11 (was 11), AvailableMemoryMB=4342 (was 4676) 2024-12-04T20:30:05,546 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=4342 2024-12-04T20:30:05,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:30:05,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:30:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:05,549 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:30:05,549 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:05,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-12-04T20:30:05,550 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:30:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T20:30:05,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742360_1536 (size=1061) 2024-12-04T20:30:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T20:30:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T20:30:05,956 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7 2024-12-04T20:30:05,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742361_1537 (size=53) 2024-12-04T20:30:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T20:30:06,361 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:30:06,361 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eec55403d95e6b84b5f2b496767f2096, disabling compactions & flushes 2024-12-04T20:30:06,361 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,361 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,361 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. after waiting 0 ms 2024-12-04T20:30:06,361 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,361 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,361 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eec55403d95e6b84b5f2b496767f2096: Waiting for close lock at 1733344206361Disabling compacts and flushes for region at 1733344206361Disabling writes for close at 1733344206361Writing region close event to WAL at 1733344206361Closed at 1733344206361 2024-12-04T20:30:06,362 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:30:06,362 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733344206362"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733344206362"}]},"ts":"1733344206362"} 2024-12-04T20:30:06,363 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:30:06,364 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:30:06,364 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344206364"}]},"ts":"1733344206364"} 2024-12-04T20:30:06,365 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T20:30:06,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, ASSIGN}] 2024-12-04T20:30:06,366 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, ASSIGN 2024-12-04T20:30:06,367 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, ASSIGN; state=OFFLINE, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=false 2024-12-04T20:30:06,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:30:06,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, ASSIGN because future has completed 2024-12-04T20:30:06,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:30:06,673 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,674 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:30:06,674 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,674 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:30:06,674 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,674 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,675 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,676 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:30:06,676 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eec55403d95e6b84b5f2b496767f2096 columnFamilyName A 2024-12-04T20:30:06,676 DEBUG [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:06,676 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(327): Store=eec55403d95e6b84b5f2b496767f2096/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:30:06,677 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,677 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:30:06,678 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eec55403d95e6b84b5f2b496767f2096 columnFamilyName B 2024-12-04T20:30:06,678 DEBUG [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:06,678 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(327): Store=eec55403d95e6b84b5f2b496767f2096/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:30:06,678 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,679 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:30:06,679 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eec55403d95e6b84b5f2b496767f2096 columnFamilyName C 2024-12-04T20:30:06,679 DEBUG [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:06,679 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(327): Store=eec55403d95e6b84b5f2b496767f2096/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:30:06,679 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,679 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,680 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,680 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,681 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,681 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T20:30:06,681 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:30:06,682 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,683 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:30:06,684 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened eec55403d95e6b84b5f2b496767f2096; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63898647, jitterRate=-0.04783596098423004}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:30:06,684 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:06,684 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for eec55403d95e6b84b5f2b496767f2096: Running coprocessor pre-open hook at 1733344206674Writing region info on filesystem at 1733344206674Initializing all the Stores at 1733344206675 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344206675Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344206675Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344206675Cleaning up temporary data from old regions at 1733344206681 (+6 ms)Running coprocessor post-open hooks at 1733344206684 (+3 ms)Region opened successfully at 1733344206684 2024-12-04T20:30:06,685 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., pid=138, masterSystemTime=1733344206671 2024-12-04T20:30:06,686 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,686 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:06,686 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:30:06,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:30:06,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-04T20:30:06,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 in 169 msec 2024-12-04T20:30:06,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-04T20:30:06,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, ASSIGN in 324 msec 2024-12-04T20:30:06,691 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:30:06,691 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344206691"}]},"ts":"1733344206691"} 2024-12-04T20:30:06,692 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T20:30:06,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:30:06,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-12-04T20:30:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T20:30:07,702 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:07,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:07,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:07,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:07,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652d66b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:07,708 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f2cf1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:07,709 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:07,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:07,712 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:30:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,43507,1733344053664 2024-12-04T20:30:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59b7e81e 2024-12-04T20:30:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:30:07,714 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:30:07,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T20:30:07,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:30:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:07,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742362_1538 (size=1097) 2024-12-04T20:30:08,130 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T20:30:08,130 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T20:30:08,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:30:08,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, REOPEN/MOVE}] 2024-12-04T20:30:08,140 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, REOPEN/MOVE 2024-12-04T20:30:08,140 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, REOPEN/MOVE because future has completed 2024-12-04T20:30:08,142 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:30:08,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:30:08,295 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,295 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:30:08,295 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing eec55403d95e6b84b5f2b496767f2096, disabling compactions & flushes 2024-12-04T20:30:08,296 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,296 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,296 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. after waiting 0 ms 2024-12-04T20:30:08,296 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,304 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T20:30:08,305 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,305 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for eec55403d95e6b84b5f2b496767f2096: Waiting for close lock at 1733344208295Running coprocessor pre-close hooks at 1733344208295Disabling compacts and flushes for region at 1733344208295Disabling writes for close at 1733344208296 (+1 ms)Writing region close event to WAL at 1733344208297 (+1 ms)Running coprocessor post-close hooks at 1733344208305 (+8 ms)Closed at 1733344208305 2024-12-04T20:30:08,305 WARN [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: eec55403d95e6b84b5f2b496767f2096 to self. 2024-12-04T20:30:08,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=CLOSED 2024-12-04T20:30:08,309 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:30:08,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-04T20:30:08,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 in 169 msec 2024-12-04T20:30:08,314 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, REOPEN/MOVE; state=CLOSED, location=c2ef38372881,45443,1733344054291; forceNewPlan=false, retain=true 2024-12-04T20:30:08,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=OPENING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, REOPEN/MOVE because future has completed 2024-12-04T20:30:08,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:30:08,631 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,631 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:30:08,632 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,632 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:30:08,632 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,632 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,635 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,636 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:30:08,636 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eec55403d95e6b84b5f2b496767f2096 columnFamilyName A 2024-12-04T20:30:08,637 DEBUG [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:08,638 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(327): Store=eec55403d95e6b84b5f2b496767f2096/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:30:08,638 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,638 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:30:08,638 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eec55403d95e6b84b5f2b496767f2096 columnFamilyName B 2024-12-04T20:30:08,639 DEBUG [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:08,639 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(327): Store=eec55403d95e6b84b5f2b496767f2096/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:30:08,639 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,639 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T20:30:08,639 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eec55403d95e6b84b5f2b496767f2096 columnFamilyName C 2024-12-04T20:30:08,639 DEBUG [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:08,640 INFO [StoreOpener-eec55403d95e6b84b5f2b496767f2096-1 {}] regionserver.HStore(327): Store=eec55403d95e6b84b5f2b496767f2096/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:30:08,640 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,640 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,640 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,641 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,642 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,642 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,642 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:30:08,643 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,643 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened eec55403d95e6b84b5f2b496767f2096; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70094704, jitterRate=0.044492483139038086}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:30:08,643 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,644 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for eec55403d95e6b84b5f2b496767f2096: Running coprocessor pre-open hook at 1733344208633Writing region info on filesystem at 1733344208633Initializing all the Stores at 1733344208634 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344208634Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344208634Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733344208634Cleaning up temporary data from old regions at 1733344208642 (+8 ms)Running coprocessor post-open hooks at 1733344208643 (+1 ms)Region opened successfully at 1733344208644 (+1 ms) 2024-12-04T20:30:08,644 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., pid=143, masterSystemTime=1733344208624 2024-12-04T20:30:08,646 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,646 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,646 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=OPEN, openSeqNum=5, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:30:08,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-12-04T20:30:08,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 in 178 msec 2024-12-04T20:30:08,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-04T20:30:08,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, REOPEN/MOVE in 510 msec 2024-12-04T20:30:08,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-04T20:30:08,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-12-04T20:30:08,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 936 msec 2024-12-04T20:30:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-04T20:30:08,655 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:08,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250f6374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb84500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9995a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b42cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,663 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcad75f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,664 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b527e01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3360db6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14597cc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42838818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205c983a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166d77a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c01737c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190b125f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513782ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,683 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b76a3f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9fa1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,688 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c701f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,690 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d5580e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43507,-1 for getting cluster id 2024-12-04T20:30:08,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:30:08,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1486f5e1-bfc3-483f-a551-5e4afcecaf44' 2024-12-04T20:30:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:30:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1486f5e1-bfc3-483f-a551-5e4afcecaf44" 2024-12-04T20:30:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f97f39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43507,-1] 2024-12-04T20:30:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:30:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:08,693 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:30:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bff7c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:30:08,694 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,694 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,694 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,695 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,695 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:08,695 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,695 DEBUG [Thread-2413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,696 DEBUG [Thread-2419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,696 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-04T20:30:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T20:30:08,697 DEBUG [Thread-2419 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,697 DEBUG [Thread-2419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,698 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:08,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,698 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,699 DEBUG [Thread-2417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:30:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:08,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:08,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,45443,1733344054291, seqNum=-1] 2024-12-04T20:30:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:30:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:30:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5] 2024-12-04T20:30:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T20:30:08,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:08,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:08,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:08,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:08,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:08,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:08,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344208724, max timestamp=1733344208728Num uniques 859; 2024-12-04T20:30:08,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:30:08,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344208724, max timestamp=1733344208728Num uniques 859; 2024-12-04T20:30:08,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:30:08,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733344208724, max timestamp=1733344208728Num uniques 859; 2024-12-04T20:30:08,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T20:30:08,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204371d4fd794aa48d0ab8bab58586749b0_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344208704/Put/seqid=0 2024-12-04T20:30:08,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742363_1539 (size=12154) 2024-12-04T20:30:08,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:08,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344268741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344268741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344268742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344268742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344268743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204371d4fd794aa48d0ab8bab58586749b0_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204371d4fd794aa48d0ab8bab58586749b0_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,748 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a34704895726432d987ffb4cea4436b9, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:08,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a34704895726432d987ffb4cea4436b9 is 175, key is test_row_0/A:col10/1733344208704/Put/seqid=0 2024-12-04T20:30:08,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742364_1540 (size=30951) 2024-12-04T20:30:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T20:30:08,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344268852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344268852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344268852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344268852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:08,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344268852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T20:30:09,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344269061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344269061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344269061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344269061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344269061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,163 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a34704895726432d987ffb4cea4436b9 2024-12-04T20:30:09,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/973147bd46624cea8e3ece16e6cea4d5 is 50, key is test_row_0/B:col10/1733344208704/Put/seqid=0 2024-12-04T20:30:09,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742365_1541 (size=9657) 2024-12-04T20:30:09,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:09,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T20:30:09,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344269372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344269372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344269372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344269372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344269372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:09,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:09,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/973147bd46624cea8e3ece16e6cea4d5 2024-12-04T20:30:09,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/e55acd0f6a9d435f83f89f190c4ef90e is 50, key is test_row_0/C:col10/1733344208704/Put/seqid=0 2024-12-04T20:30:09,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742366_1542 (size=9657) 2024-12-04T20:30:09,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:09,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:09,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:09,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:09,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T20:30:09,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344269881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344269881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344269881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344269881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344269881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:09,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:09,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:09,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:09,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:10,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/e55acd0f6a9d435f83f89f190c4ef90e 2024-12-04T20:30:10,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a34704895726432d987ffb4cea4436b9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a34704895726432d987ffb4cea4436b9 2024-12-04T20:30:10,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a34704895726432d987ffb4cea4436b9, entries=150, sequenceid=14, filesize=30.2 K 2024-12-04T20:30:10,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/973147bd46624cea8e3ece16e6cea4d5 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/973147bd46624cea8e3ece16e6cea4d5 2024-12-04T20:30:10,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/973147bd46624cea8e3ece16e6cea4d5, entries=100, sequenceid=14, filesize=9.4 K 2024-12-04T20:30:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/e55acd0f6a9d435f83f89f190c4ef90e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/e55acd0f6a9d435f83f89f190c4ef90e 2024-12-04T20:30:10,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/e55acd0f6a9d435f83f89f190c4ef90e, entries=100, sequenceid=14, filesize=9.4 K 2024-12-04T20:30:10,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eec55403d95e6b84b5f2b496767f2096 in 1309ms, sequenceid=14, compaction requested=false 2024-12-04T20:30:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:10,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T20:30:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:10,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T20:30:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:10,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045a65b5d394ee46d2b7247ae9679b0f14_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344208741/Put/seqid=0 2024-12-04T20:30:10,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742367_1543 (size=12154) 2024-12-04T20:30:10,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:10,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045a65b5d394ee46d2b7247ae9679b0f14_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045a65b5d394ee46d2b7247ae9679b0f14_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/47c830875d474bc48f53e7db022ae662, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/47c830875d474bc48f53e7db022ae662 is 175, key is test_row_0/A:col10/1733344208741/Put/seqid=0 2024-12-04T20:30:10,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742368_1544 (size=30955) 2024-12-04T20:30:10,094 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/47c830875d474bc48f53e7db022ae662 2024-12-04T20:30:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/81e9a9e9fe784483a0541357eaec9922 is 50, key is test_row_0/B:col10/1733344208741/Put/seqid=0 2024-12-04T20:30:10,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742369_1545 (size=12001) 2024-12-04T20:30:10,506 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/81e9a9e9fe784483a0541357eaec9922 2024-12-04T20:30:10,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/9abf45f7566345b196229fc3c42d54f4 is 50, key is test_row_0/C:col10/1733344208741/Put/seqid=0 2024-12-04T20:30:10,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742370_1546 (size=12001) 2024-12-04T20:30:10,609 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:30:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T20:30:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:10,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344270904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344270905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344270906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344270906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344270907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:10,951 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/9abf45f7566345b196229fc3c42d54f4 2024-12-04T20:30:10,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/47c830875d474bc48f53e7db022ae662 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/47c830875d474bc48f53e7db022ae662 2024-12-04T20:30:10,957 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/47c830875d474bc48f53e7db022ae662, entries=150, sequenceid=51, filesize=30.2 K 2024-12-04T20:30:10,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/81e9a9e9fe784483a0541357eaec9922 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/81e9a9e9fe784483a0541357eaec9922 2024-12-04T20:30:10,961 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/81e9a9e9fe784483a0541357eaec9922, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T20:30:10,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/9abf45f7566345b196229fc3c42d54f4 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9abf45f7566345b196229fc3c42d54f4 2024-12-04T20:30:10,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9abf45f7566345b196229fc3c42d54f4, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T20:30:10,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for eec55403d95e6b84b5f2b496767f2096 in 888ms, sequenceid=51, compaction requested=false 2024-12-04T20:30:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-04T20:30:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-04T20:30:10,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-04T20:30:10,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-12-04T20:30:10,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.2720 sec 2024-12-04T20:30:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:11,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:30:11,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:11,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:11,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:11,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:11,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:11,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:11,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ac2ab56aeffc4dfe92a44286017cb180_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344211014/Put/seqid=0 2024-12-04T20:30:11,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344271029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742371_1547 (size=17034) 2024-12-04T20:30:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344271030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344271031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344271032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344271033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344271141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344271141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344271141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344271142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344271142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344271351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344271352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344271352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344271352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344271352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,433 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:11,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ac2ab56aeffc4dfe92a44286017cb180_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ac2ab56aeffc4dfe92a44286017cb180_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:11,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/f6dbfa704f724cc8afad636010db8ea7, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:11,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/f6dbfa704f724cc8afad636010db8ea7 is 175, key is test_row_0/A:col10/1733344211014/Put/seqid=0 2024-12-04T20:30:11,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742372_1548 (size=48139) 2024-12-04T20:30:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344271661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344271661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344271661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344271661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344271661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:11,840 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/f6dbfa704f724cc8afad636010db8ea7 2024-12-04T20:30:11,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/1bb2804d13af4794906f53d06f93fe7f is 50, key is test_row_0/B:col10/1733344211014/Put/seqid=0 2024-12-04T20:30:11,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742373_1549 (size=12001) 2024-12-04T20:30:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344272171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344272171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:12,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344272171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344272172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344272172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:12,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/1bb2804d13af4794906f53d06f93fe7f 2024-12-04T20:30:12,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/df060a94dc254e4e87c24688cfa9dc46 is 50, key is test_row_0/C:col10/1733344211014/Put/seqid=0 2024-12-04T20:30:12,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742374_1550 (size=12001) 2024-12-04T20:30:12,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/df060a94dc254e4e87c24688cfa9dc46 2024-12-04T20:30:12,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/f6dbfa704f724cc8afad636010db8ea7 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/f6dbfa704f724cc8afad636010db8ea7 2024-12-04T20:30:12,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/f6dbfa704f724cc8afad636010db8ea7, entries=250, sequenceid=65, filesize=47.0 K 2024-12-04T20:30:12,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/1bb2804d13af4794906f53d06f93fe7f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/1bb2804d13af4794906f53d06f93fe7f 2024-12-04T20:30:12,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/1bb2804d13af4794906f53d06f93fe7f, entries=150, sequenceid=65, filesize=11.7 K 2024-12-04T20:30:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/df060a94dc254e4e87c24688cfa9dc46 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/df060a94dc254e4e87c24688cfa9dc46 2024-12-04T20:30:12,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/df060a94dc254e4e87c24688cfa9dc46, entries=150, sequenceid=65, filesize=11.7 K 2024-12-04T20:30:12,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for eec55403d95e6b84b5f2b496767f2096 in 1263ms, sequenceid=65, compaction requested=true 2024-12-04T20:30:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:30:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:12,277 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:30:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:12,277 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:30:12,278 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:12,278 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/B is initiating minor compaction (all files) 2024-12-04T20:30:12,278 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/B in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:12,278 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/973147bd46624cea8e3ece16e6cea4d5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/81e9a9e9fe784483a0541357eaec9922, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/1bb2804d13af4794906f53d06f93fe7f] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=32.9 K 2024-12-04T20:30:12,278 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 973147bd46624cea8e3ece16e6cea4d5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733344208704 2024-12-04T20:30:12,279 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e9a9e9fe784483a0541357eaec9922, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733344208740 2024-12-04T20:30:12,279 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:12,279 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/A is initiating minor compaction (all files) 2024-12-04T20:30:12,279 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/A in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:12,279 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a34704895726432d987ffb4cea4436b9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/47c830875d474bc48f53e7db022ae662, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/f6dbfa704f724cc8afad636010db8ea7] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=107.5 K 2024-12-04T20:30:12,279 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb2804d13af4794906f53d06f93fe7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733344210899 2024-12-04T20:30:12,279 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:12,279 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a34704895726432d987ffb4cea4436b9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/47c830875d474bc48f53e7db022ae662, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/f6dbfa704f724cc8afad636010db8ea7] 2024-12-04T20:30:12,279 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting a34704895726432d987ffb4cea4436b9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733344208704 2024-12-04T20:30:12,280 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47c830875d474bc48f53e7db022ae662, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733344208740 2024-12-04T20:30:12,281 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6dbfa704f724cc8afad636010db8ea7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733344210899 2024-12-04T20:30:12,288 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:12,288 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/7c67897da7244f8e9237ff33f9602655 is 50, key is test_row_0/B:col10/1733344211014/Put/seqid=0 2024-12-04T20:30:12,293 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:12,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742375_1551 (size=12104) 2024-12-04T20:30:12,295 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204631137e7c2a0411f8633bc7b99b66bd8_eec55403d95e6b84b5f2b496767f2096 store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:12,297 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204631137e7c2a0411f8633bc7b99b66bd8_eec55403d95e6b84b5f2b496767f2096, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:12,297 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204631137e7c2a0411f8633bc7b99b66bd8_eec55403d95e6b84b5f2b496767f2096 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:12,301 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/7c67897da7244f8e9237ff33f9602655 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7c67897da7244f8e9237ff33f9602655 2024-12-04T20:30:12,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742376_1552 (size=4469) 2024-12-04T20:30:12,303 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#A#compaction#466 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:12,304 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/beb02c381e214d9d8f6014360be16e33 is 175, key is test_row_0/A:col10/1733344211014/Put/seqid=0 2024-12-04T20:30:12,316 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/B of eec55403d95e6b84b5f2b496767f2096 into 7c67897da7244f8e9237ff33f9602655(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:12,316 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:12,316 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/B, priority=13, startTime=1733344212277; duration=0sec 2024-12-04T20:30:12,316 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:12,316 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:B 2024-12-04T20:30:12,316 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:12,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742377_1553 (size=31058) 2024-12-04T20:30:12,317 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:12,318 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/C is initiating minor compaction (all files) 2024-12-04T20:30:12,318 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/C in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:12,318 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/e55acd0f6a9d435f83f89f190c4ef90e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9abf45f7566345b196229fc3c42d54f4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/df060a94dc254e4e87c24688cfa9dc46] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=32.9 K 2024-12-04T20:30:12,318 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting e55acd0f6a9d435f83f89f190c4ef90e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733344208704 2024-12-04T20:30:12,318 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9abf45f7566345b196229fc3c42d54f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733344208740 2024-12-04T20:30:12,319 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting df060a94dc254e4e87c24688cfa9dc46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733344210899 2024-12-04T20:30:12,321 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/beb02c381e214d9d8f6014360be16e33 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/beb02c381e214d9d8f6014360be16e33 2024-12-04T20:30:12,326 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/A of eec55403d95e6b84b5f2b496767f2096 into beb02c381e214d9d8f6014360be16e33(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:12,326 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:12,326 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/A, priority=13, startTime=1733344212277; duration=0sec 2024-12-04T20:30:12,326 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:12,326 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:A 2024-12-04T20:30:12,331 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:12,331 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/8ef63d51dbb3424ea1284c4f07052b9d is 50, key is test_row_0/C:col10/1733344211014/Put/seqid=0 2024-12-04T20:30:12,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742378_1554 (size=12104) 2024-12-04T20:30:12,741 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/8ef63d51dbb3424ea1284c4f07052b9d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8ef63d51dbb3424ea1284c4f07052b9d 2024-12-04T20:30:12,744 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/C of eec55403d95e6b84b5f2b496767f2096 into 8ef63d51dbb3424ea1284c4f07052b9d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:12,744 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:12,745 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/C, priority=13, startTime=1733344212278; duration=0sec 2024-12-04T20:30:12,745 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:12,745 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:C 2024-12-04T20:30:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T20:30:12,862 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:12,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-04T20:30:12,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T20:30:12,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:12,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T20:30:13,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T20:30:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:13,018 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T20:30:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:13,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120485776ee7be414130acbd654f21a68822_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344211029/Put/seqid=0 2024-12-04T20:30:13,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742379_1555 (size=12154) 2024-12-04T20:30:13,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:13,036 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120485776ee7be414130acbd654f21a68822_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120485776ee7be414130acbd654f21a68822_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:13,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a670be7bc72048f6975f103686d74599, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:13,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a670be7bc72048f6975f103686d74599 is 175, key is test_row_0/A:col10/1733344211029/Put/seqid=0 2024-12-04T20:30:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742380_1556 (size=30955) 2024-12-04T20:30:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T20:30:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:13,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344273189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344273191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344273191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344273191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344273192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344273302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344273302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344273302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344273302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344273302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a670be7bc72048f6975f103686d74599 2024-12-04T20:30:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/4bc25cdb36c04b33b961a280150d8cf6 is 50, key is test_row_0/B:col10/1733344211029/Put/seqid=0 2024-12-04T20:30:13,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742381_1557 (size=12001) 2024-12-04T20:30:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T20:30:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344273511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344273512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344273512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344273512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344273512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344273821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344273822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344273822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344273822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344273822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:13,852 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/4bc25cdb36c04b33b961a280150d8cf6 2024-12-04T20:30:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/c3e2a0d2ef3b476792f806207b229d39 is 50, key is test_row_0/C:col10/1733344211029/Put/seqid=0 2024-12-04T20:30:13,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742382_1558 (size=12001) 2024-12-04T20:30:13,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/c3e2a0d2ef3b476792f806207b229d39 2024-12-04T20:30:13,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/a670be7bc72048f6975f103686d74599 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a670be7bc72048f6975f103686d74599 2024-12-04T20:30:13,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a670be7bc72048f6975f103686d74599, entries=150, sequenceid=87, filesize=30.2 K 2024-12-04T20:30:13,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/4bc25cdb36c04b33b961a280150d8cf6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4bc25cdb36c04b33b961a280150d8cf6 2024-12-04T20:30:13,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4bc25cdb36c04b33b961a280150d8cf6, entries=150, sequenceid=87, filesize=11.7 K 2024-12-04T20:30:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/c3e2a0d2ef3b476792f806207b229d39 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c3e2a0d2ef3b476792f806207b229d39 2024-12-04T20:30:13,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c3e2a0d2ef3b476792f806207b229d39, entries=150, sequenceid=87, filesize=11.7 K 2024-12-04T20:30:13,888 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eec55403d95e6b84b5f2b496767f2096 in 871ms, sequenceid=87, compaction requested=false 2024-12-04T20:30:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-04T20:30:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-04T20:30:13,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-04T20:30:13,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0240 sec 2024-12-04T20:30:13,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.0300 sec 2024-12-04T20:30:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T20:30:14,002 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:14,002 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-04T20:30:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T20:30:14,004 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:14,005 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:14,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T20:30:14,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T20:30:14,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:14,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:30:14,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:14,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:14,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:14,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:14,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e2c7458fa499462c9319c4eabddd5a42_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344213191/Put/seqid=0 2024-12-04T20:30:14,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742383_1559 (size=12154) 2024-12-04T20:30:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T20:30:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:14,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:14,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344274345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344274348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344274349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344274350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344274351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344274461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344274461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344274461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344274462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344274462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:14,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e2c7458fa499462c9319c4eabddd5a42_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e2c7458fa499462c9319c4eabddd5a42_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/67f2dc3668e94e16bc19978e580323e6, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/67f2dc3668e94e16bc19978e580323e6 is 175, key is test_row_0/A:col10/1733344213191/Put/seqid=0 2024-12-04T20:30:14,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742384_1560 (size=30955) 2024-12-04T20:30:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T20:30:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344274671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344274671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344274672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344274672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344274672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/67f2dc3668e94e16bc19978e580323e6 2024-12-04T20:30:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344274981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344274981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344274982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344274982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344274982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/6f9b9b8e452b4c48bf06c46606d02227 is 50, key is test_row_0/B:col10/1733344213191/Put/seqid=0 2024-12-04T20:30:14,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742385_1561 (size=12001) 2024-12-04T20:30:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T20:30:15,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/6f9b9b8e452b4c48bf06c46606d02227 2024-12-04T20:30:15,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/890d2b5debcc4b28ac4e99da34ac8502 is 50, key is test_row_0/C:col10/1733344213191/Put/seqid=0 2024-12-04T20:30:15,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742386_1562 (size=12001) 2024-12-04T20:30:15,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344275492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:15,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344275492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344275492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344275492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344275492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:15,807 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/890d2b5debcc4b28ac4e99da34ac8502 2024-12-04T20:30:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/67f2dc3668e94e16bc19978e580323e6 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/67f2dc3668e94e16bc19978e580323e6 2024-12-04T20:30:15,813 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/67f2dc3668e94e16bc19978e580323e6, entries=150, sequenceid=100, filesize=30.2 K 2024-12-04T20:30:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/6f9b9b8e452b4c48bf06c46606d02227 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6f9b9b8e452b4c48bf06c46606d02227 2024-12-04T20:30:15,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6f9b9b8e452b4c48bf06c46606d02227, entries=150, sequenceid=100, filesize=11.7 K 2024-12-04T20:30:15,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/890d2b5debcc4b28ac4e99da34ac8502 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/890d2b5debcc4b28ac4e99da34ac8502 2024-12-04T20:30:15,821 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/890d2b5debcc4b28ac4e99da34ac8502, entries=150, sequenceid=100, filesize=11.7 K 2024-12-04T20:30:15,822 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for eec55403d95e6b84b5f2b496767f2096 in 1664ms, sequenceid=100, compaction requested=true 2024-12-04T20:30:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-04T20:30:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-04T20:30:15,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-04T20:30:15,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-12-04T20:30:15,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.8230 sec 2024-12-04T20:30:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T20:30:16,151 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:16,152 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-04T20:30:16,154 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T20:30:16,155 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:16,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T20:30:16,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T20:30:16,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:16,309 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T20:30:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:16,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a37a7918785245648acdef09749997a6_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344214349/Put/seqid=0 2024-12-04T20:30:16,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742387_1563 (size=12154) 2024-12-04T20:30:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T20:30:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:16,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:16,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344276512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344276514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344276515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344276519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344276519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344276621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344276621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344276632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344276632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344276632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:16,756 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a37a7918785245648acdef09749997a6_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a37a7918785245648acdef09749997a6_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:16,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/77ddeb6d392d473187120fa5e4f0ca19, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:16,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/77ddeb6d392d473187120fa5e4f0ca19 is 175, key is test_row_0/A:col10/1733344214349/Put/seqid=0 2024-12-04T20:30:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T20:30:16,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742388_1564 (size=30955) 2024-12-04T20:30:16,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344276832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344276832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344276841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344276841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:16,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344276841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344277142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344277142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344277151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344277151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344277151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/77ddeb6d392d473187120fa5e4f0ca19 2024-12-04T20:30:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/bf8ab3f8b47847588f5bb69baf0180eb is 50, key is test_row_0/B:col10/1733344214349/Put/seqid=0 2024-12-04T20:30:17,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742389_1565 (size=12001) 2024-12-04T20:30:17,231 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/bf8ab3f8b47847588f5bb69baf0180eb 2024-12-04T20:30:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/64809aafb47f4ab8867a7e517b59fbc1 is 50, key is test_row_0/C:col10/1733344214349/Put/seqid=0 2024-12-04T20:30:17,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742390_1566 (size=12001) 2024-12-04T20:30:17,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/64809aafb47f4ab8867a7e517b59fbc1 2024-12-04T20:30:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/77ddeb6d392d473187120fa5e4f0ca19 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/77ddeb6d392d473187120fa5e4f0ca19 2024-12-04T20:30:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T20:30:17,295 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/77ddeb6d392d473187120fa5e4f0ca19, entries=150, sequenceid=120, filesize=30.2 K 2024-12-04T20:30:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/bf8ab3f8b47847588f5bb69baf0180eb as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf8ab3f8b47847588f5bb69baf0180eb 2024-12-04T20:30:17,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf8ab3f8b47847588f5bb69baf0180eb, entries=150, sequenceid=120, filesize=11.7 K 2024-12-04T20:30:17,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/64809aafb47f4ab8867a7e517b59fbc1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/64809aafb47f4ab8867a7e517b59fbc1 2024-12-04T20:30:17,317 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/64809aafb47f4ab8867a7e517b59fbc1, entries=150, sequenceid=120, filesize=11.7 K 2024-12-04T20:30:17,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for eec55403d95e6b84b5f2b496767f2096 in 1011ms, sequenceid=120, compaction requested=true 2024-12-04T20:30:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-04T20:30:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-04T20:30:17,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-04T20:30:17,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1650 sec 2024-12-04T20:30:17,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.1720 sec 2024-12-04T20:30:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:17,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T20:30:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:17,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cf818dd89b594e1fbaaa7a56a60aef41_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344216503/Put/seqid=0 2024-12-04T20:30:17,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742391_1567 (size=17234) 2024-12-04T20:30:17,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344277672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344277672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344277674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344277677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344277677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344277783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344277783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344277783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344277792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344277792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344277992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344277992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344277992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344278001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344278001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,068 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:18,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cf818dd89b594e1fbaaa7a56a60aef41_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cf818dd89b594e1fbaaa7a56a60aef41_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:18,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/92b4dee409af4ef886e62921895c2ffe, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:18,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/92b4dee409af4ef886e62921895c2ffe is 175, key is test_row_0/A:col10/1733344216503/Put/seqid=0 2024-12-04T20:30:18,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742392_1568 (size=48339) 2024-12-04T20:30:18,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/92b4dee409af4ef886e62921895c2ffe 2024-12-04T20:30:18,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/4ef6ac9540ac48f796778a036d5a6ba1 is 50, key is test_row_0/B:col10/1733344216503/Put/seqid=0 2024-12-04T20:30:18,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742393_1569 (size=12101) 2024-12-04T20:30:18,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/4ef6ac9540ac48f796778a036d5a6ba1 2024-12-04T20:30:18,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/b3f7339fe8a54a34973f146c0d08c5df is 50, key is test_row_0/C:col10/1733344216503/Put/seqid=0 2024-12-04T20:30:18,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742394_1570 (size=12101) 2024-12-04T20:30:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T20:30:18,302 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:18,303 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-04T20:30:18,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T20:30:18,305 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:18,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344278302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344278302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344278302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344278312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344278312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T20:30:18,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T20:30:18,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:18,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:18,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:18,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:18,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:18,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/b3f7339fe8a54a34973f146c0d08c5df 2024-12-04T20:30:18,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/92b4dee409af4ef886e62921895c2ffe as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/92b4dee409af4ef886e62921895c2ffe 2024-12-04T20:30:18,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/92b4dee409af4ef886e62921895c2ffe, entries=250, sequenceid=135, filesize=47.2 K 2024-12-04T20:30:18,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/4ef6ac9540ac48f796778a036d5a6ba1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4ef6ac9540ac48f796778a036d5a6ba1 2024-12-04T20:30:18,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4ef6ac9540ac48f796778a036d5a6ba1, entries=150, sequenceid=135, filesize=11.8 K 2024-12-04T20:30:18,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/b3f7339fe8a54a34973f146c0d08c5df as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/b3f7339fe8a54a34973f146c0d08c5df 2024-12-04T20:30:18,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/b3f7339fe8a54a34973f146c0d08c5df, entries=150, sequenceid=135, filesize=11.8 K 2024-12-04T20:30:18,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for eec55403d95e6b84b5f2b496767f2096 in 945ms, sequenceid=135, compaction requested=true 2024-12-04T20:30:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:30:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:18,599 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T20:30:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:30:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:18,599 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T20:30:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:30:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:18,600 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60208 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T20:30:18,600 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172262 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T20:30:18,600 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/B is initiating minor compaction (all files) 2024-12-04T20:30:18,600 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/A is initiating minor compaction (all files) 2024-12-04T20:30:18,600 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/A in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:18,600 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/B in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:18,600 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7c67897da7244f8e9237ff33f9602655, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4bc25cdb36c04b33b961a280150d8cf6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6f9b9b8e452b4c48bf06c46606d02227, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf8ab3f8b47847588f5bb69baf0180eb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4ef6ac9540ac48f796778a036d5a6ba1] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=58.8 K 2024-12-04T20:30:18,600 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/beb02c381e214d9d8f6014360be16e33, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a670be7bc72048f6975f103686d74599, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/67f2dc3668e94e16bc19978e580323e6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/77ddeb6d392d473187120fa5e4f0ca19, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/92b4dee409af4ef886e62921895c2ffe] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=168.2 K 2024-12-04T20:30:18,600 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:18,601 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/beb02c381e214d9d8f6014360be16e33, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a670be7bc72048f6975f103686d74599, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/67f2dc3668e94e16bc19978e580323e6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/77ddeb6d392d473187120fa5e4f0ca19, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/92b4dee409af4ef886e62921895c2ffe] 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c67897da7244f8e9237ff33f9602655, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733344210899 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb02c381e214d9d8f6014360be16e33, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733344210899 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bc25cdb36c04b33b961a280150d8cf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733344211028 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting a670be7bc72048f6975f103686d74599, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733344211028 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f9b9b8e452b4c48bf06c46606d02227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733344213183 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f2dc3668e94e16bc19978e580323e6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733344213183 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ddeb6d392d473187120fa5e4f0ca19, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733344214347 2024-12-04T20:30:18,602 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting bf8ab3f8b47847588f5bb69baf0180eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733344214347 2024-12-04T20:30:18,603 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b4dee409af4ef886e62921895c2ffe, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733344216503 2024-12-04T20:30:18,603 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ef6ac9540ac48f796778a036d5a6ba1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733344216503 2024-12-04T20:30:18,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T20:30:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:18,614 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T20:30:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:18,617 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#B#compaction#480 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:18,618 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/2f10d2d435a8423b8d91328fea5484d3 is 50, key is test_row_0/B:col10/1733344216503/Put/seqid=0 2024-12-04T20:30:18,619 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T20:30:18,661 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204f2701e72a93b4b0f827dd653b83593f3_eec55403d95e6b84b5f2b496767f2096 store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:18,664 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204f2701e72a93b4b0f827dd653b83593f3_eec55403d95e6b84b5f2b496767f2096, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:18,664 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f2701e72a93b4b0f827dd653b83593f3_eec55403d95e6b84b5f2b496767f2096 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:18,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204eeb690563d204c1e98cc5e2b504961dd_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344217676/Put/seqid=0 2024-12-04T20:30:18,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742395_1571 (size=12375) 2024-12-04T20:30:18,685 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/2f10d2d435a8423b8d91328fea5484d3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/2f10d2d435a8423b8d91328fea5484d3 2024-12-04T20:30:18,690 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/B of eec55403d95e6b84b5f2b496767f2096 into 2f10d2d435a8423b8d91328fea5484d3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:18,690 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:18,690 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/B, priority=11, startTime=1733344218599; duration=0sec 2024-12-04T20:30:18,691 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:18,691 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:B 2024-12-04T20:30:18,691 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T20:30:18,692 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60208 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T20:30:18,693 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/C is initiating minor compaction (all files) 2024-12-04T20:30:18,693 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/C in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:18,694 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8ef63d51dbb3424ea1284c4f07052b9d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c3e2a0d2ef3b476792f806207b229d39, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/890d2b5debcc4b28ac4e99da34ac8502, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/64809aafb47f4ab8867a7e517b59fbc1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/b3f7339fe8a54a34973f146c0d08c5df] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=58.8 K 2024-12-04T20:30:18,694 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ef63d51dbb3424ea1284c4f07052b9d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733344210899 2024-12-04T20:30:18,694 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e2a0d2ef3b476792f806207b229d39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733344211028 2024-12-04T20:30:18,695 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 890d2b5debcc4b28ac4e99da34ac8502, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733344213183 2024-12-04T20:30:18,695 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 64809aafb47f4ab8867a7e517b59fbc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733344214347 2024-12-04T20:30:18,695 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f7339fe8a54a34973f146c0d08c5df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733344216503 2024-12-04T20:30:18,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742396_1572 (size=4469) 2024-12-04T20:30:18,716 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#C#compaction#483 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:18,716 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/9d5fa5c7ab434f718c4848470f2df6f2 is 50, key is test_row_0/C:col10/1733344216503/Put/seqid=0 2024-12-04T20:30:18,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742397_1573 (size=12304) 2024-12-04T20:30:18,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:18,727 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204eeb690563d204c1e98cc5e2b504961dd_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204eeb690563d204c1e98cc5e2b504961dd_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fd1042c1befe40869cdae69db17d4d04, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fd1042c1befe40869cdae69db17d4d04 is 175, key is test_row_0/A:col10/1733344217676/Put/seqid=0 2024-12-04T20:30:18,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742398_1574 (size=12375) 2024-12-04T20:30:18,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742399_1575 (size=31105) 2024-12-04T20:30:18,751 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/9d5fa5c7ab434f718c4848470f2df6f2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9d5fa5c7ab434f718c4848470f2df6f2 2024-12-04T20:30:18,755 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/C of eec55403d95e6b84b5f2b496767f2096 into 9d5fa5c7ab434f718c4848470f2df6f2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:18,755 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:18,755 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/C, priority=11, startTime=1733344218599; duration=0sec 2024-12-04T20:30:18,755 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:18,755 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:C 2024-12-04T20:30:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:18,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344278828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344278832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344278835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344278835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344278837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T20:30:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344278941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344278942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344278952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344278952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344278952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,112 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#A#compaction#481 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:19,112 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/bb244f3e4da447618e9b67e7aab14b47 is 175, key is test_row_0/A:col10/1733344216503/Put/seqid=0 2024-12-04T20:30:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742400_1576 (size=31329) 2024-12-04T20:30:19,130 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/bb244f3e4da447618e9b67e7aab14b47 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/bb244f3e4da447618e9b67e7aab14b47 2024-12-04T20:30:19,134 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/A of eec55403d95e6b84b5f2b496767f2096 into bb244f3e4da447618e9b67e7aab14b47(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:19,134 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:19,134 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/A, priority=11, startTime=1733344218599; duration=0sec 2024-12-04T20:30:19,134 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:19,134 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:A 2024-12-04T20:30:19,149 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fd1042c1befe40869cdae69db17d4d04 2024-12-04T20:30:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344279151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344279151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/79853bb3fe2d41c89425d8b830b0c8ae is 50, key is test_row_0/B:col10/1733344217676/Put/seqid=0 2024-12-04T20:30:19,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344279162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742401_1577 (size=12151) 2024-12-04T20:30:19,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344279162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344279162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T20:30:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344279462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344279462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344279472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344279472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344279472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,563 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/79853bb3fe2d41c89425d8b830b0c8ae 2024-12-04T20:30:19,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/f0d0a1e978d3449290aef3930b245861 is 50, key is test_row_0/C:col10/1733344217676/Put/seqid=0 2024-12-04T20:30:19,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742402_1578 (size=12151) 2024-12-04T20:30:19,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344279971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344279971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344279981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344279982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/f0d0a1e978d3449290aef3930b245861 2024-12-04T20:30:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344279982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fd1042c1befe40869cdae69db17d4d04 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fd1042c1befe40869cdae69db17d4d04 2024-12-04T20:30:20,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fd1042c1befe40869cdae69db17d4d04, entries=150, sequenceid=153, filesize=30.4 K 2024-12-04T20:30:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/79853bb3fe2d41c89425d8b830b0c8ae as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/79853bb3fe2d41c89425d8b830b0c8ae 2024-12-04T20:30:20,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/79853bb3fe2d41c89425d8b830b0c8ae, entries=150, sequenceid=153, filesize=11.9 K 2024-12-04T20:30:20,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/f0d0a1e978d3449290aef3930b245861 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0d0a1e978d3449290aef3930b245861 2024-12-04T20:30:20,010 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0d0a1e978d3449290aef3930b245861, entries=150, sequenceid=153, filesize=11.9 K 2024-12-04T20:30:20,011 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for eec55403d95e6b84b5f2b496767f2096 in 1397ms, sequenceid=153, compaction requested=false 2024-12-04T20:30:20,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:20,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:20,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-04T20:30:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-04T20:30:20,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-04T20:30:20,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7070 sec 2024-12-04T20:30:20,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7120 sec 2024-12-04T20:30:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T20:30:20,452 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:20,452 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-04T20:30:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T20:30:20,454 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:20,455 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:20,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T20:30:20,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T20:30:20,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:20,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T20:30:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:20,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e6e2c61fb02044fdbd3b04eb2365833e_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344218832/Put/seqid=0 2024-12-04T20:30:20,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742403_1579 (size=12304) 2024-12-04T20:30:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T20:30:20,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344281001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344281002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344281002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,004 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in eec55403d95e6b84b5f2b496767f2096 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,005 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in eec55403d95e6b84b5f2b496767f2096 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344281004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344281004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:21,041 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e6e2c61fb02044fdbd3b04eb2365833e_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e6e2c61fb02044fdbd3b04eb2365833e_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/08ede8cb189340198b329c6181c90734, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:21,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/08ede8cb189340198b329c6181c90734 is 175, key is test_row_0/A:col10/1733344218832/Put/seqid=0 2024-12-04T20:30:21,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742404_1580 (size=31105) 2024-12-04T20:30:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T20:30:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344281111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344281111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344281111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344281321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344281321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344281321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,445 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/08ede8cb189340198b329c6181c90734 2024-12-04T20:30:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/585969f43ac743d6bb72556680621d93 is 50, key is test_row_0/B:col10/1733344218832/Put/seqid=0 2024-12-04T20:30:21,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742405_1581 (size=12151) 2024-12-04T20:30:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T20:30:21,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344281632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344281632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344281632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:21,853 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/585969f43ac743d6bb72556680621d93 2024-12-04T20:30:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/8e52feed69ce4348a32dfb3a20dd194e is 50, key is test_row_0/C:col10/1733344218832/Put/seqid=0 2024-12-04T20:30:21,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742406_1582 (size=12151) 2024-12-04T20:30:21,864 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/8e52feed69ce4348a32dfb3a20dd194e 2024-12-04T20:30:21,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/08ede8cb189340198b329c6181c90734 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/08ede8cb189340198b329c6181c90734 2024-12-04T20:30:21,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/08ede8cb189340198b329c6181c90734, entries=150, sequenceid=171, filesize=30.4 K 2024-12-04T20:30:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/585969f43ac743d6bb72556680621d93 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/585969f43ac743d6bb72556680621d93 2024-12-04T20:30:21,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/585969f43ac743d6bb72556680621d93, entries=150, sequenceid=171, filesize=11.9 K 2024-12-04T20:30:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/8e52feed69ce4348a32dfb3a20dd194e as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8e52feed69ce4348a32dfb3a20dd194e 2024-12-04T20:30:21,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8e52feed69ce4348a32dfb3a20dd194e, entries=150, sequenceid=171, filesize=11.9 K 2024-12-04T20:30:21,885 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for eec55403d95e6b84b5f2b496767f2096 in 1277ms, sequenceid=171, compaction requested=true 2024-12-04T20:30:21,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:21,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:21,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-04T20:30:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-04T20:30:21,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-04T20:30:21,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-12-04T20:30:21,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4370 sec 2024-12-04T20:30:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:22,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T20:30:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:22,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c2633ed00f404d2f96ce49650d28e0df_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344221003/Put/seqid=0 2024-12-04T20:30:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742407_1583 (size=14794) 2024-12-04T20:30:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344282164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344282165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344282165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344282271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344282271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344282271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344282481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344282481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344282481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:22,557 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c2633ed00f404d2f96ce49650d28e0df_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c2633ed00f404d2f96ce49650d28e0df_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:22,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/53706001c81540cb891df874f44d49ca, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:22,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/53706001c81540cb891df874f44d49ca is 175, key is test_row_0/A:col10/1733344221003/Put/seqid=0 2024-12-04T20:30:22,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742408_1584 (size=39749) 2024-12-04T20:30:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T20:30:22,601 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:22,602 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-04T20:30:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T20:30:22,603 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:22,604 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:22,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T20:30:22,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:22,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:22,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344282792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344282792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344282792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:22,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:22,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T20:30:22,961 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/53706001c81540cb891df874f44d49ca 2024-12-04T20:30:22,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/cd3f1afd1b30443bafb6511bb0edd99f is 50, key is test_row_0/B:col10/1733344221003/Put/seqid=0 2024-12-04T20:30:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742409_1585 (size=12151) 2024-12-04T20:30:23,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344283012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:23,014 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in eec55403d95e6b84b5f2b496767f2096 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:30:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344283031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:23,033 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in eec55403d95e6b84b5f2b496767f2096 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:30:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:23,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T20:30:23,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344283301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344283301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:23,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344283301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:23,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/cd3f1afd1b30443bafb6511bb0edd99f 2024-12-04T20:30:23,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/23be68490be34b4887e4b6747276d4ca is 50, key is test_row_0/C:col10/1733344221003/Put/seqid=0 2024-12-04T20:30:23,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742410_1586 (size=12151) 2024-12-04T20:30:23,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T20:30:23,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/23be68490be34b4887e4b6747276d4ca 2024-12-04T20:30:23,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/53706001c81540cb891df874f44d49ca as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/53706001c81540cb891df874f44d49ca 2024-12-04T20:30:23,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/53706001c81540cb891df874f44d49ca, entries=200, sequenceid=191, filesize=38.8 K 2024-12-04T20:30:23,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/cd3f1afd1b30443bafb6511bb0edd99f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/cd3f1afd1b30443bafb6511bb0edd99f 2024-12-04T20:30:23,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/cd3f1afd1b30443bafb6511bb0edd99f, entries=150, sequenceid=191, filesize=11.9 K 2024-12-04T20:30:23,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/23be68490be34b4887e4b6747276d4ca as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/23be68490be34b4887e4b6747276d4ca 2024-12-04T20:30:23,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/23be68490be34b4887e4b6747276d4ca, entries=150, sequenceid=191, filesize=11.9 K 2024-12-04T20:30:23,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for eec55403d95e6b84b5f2b496767f2096 in 1676ms, sequenceid=191, compaction requested=true 2024-12-04T20:30:23,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:23,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:30:23,820 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:30:23,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:23,820 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:30:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:30:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:30:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:23,821 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133288 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:30:23,821 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/A is initiating minor compaction (all files) 2024-12-04T20:30:23,821 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:30:23,821 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/B is initiating minor compaction (all files) 2024-12-04T20:30:23,822 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/A in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,822 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/B in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,822 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/bb244f3e4da447618e9b67e7aab14b47, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fd1042c1befe40869cdae69db17d4d04, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/08ede8cb189340198b329c6181c90734, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/53706001c81540cb891df874f44d49ca] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=130.2 K 2024-12-04T20:30:23,822 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/2f10d2d435a8423b8d91328fea5484d3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/79853bb3fe2d41c89425d8b830b0c8ae, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/585969f43ac743d6bb72556680621d93, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/cd3f1afd1b30443bafb6511bb0edd99f] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=47.7 K 2024-12-04T20:30:23,822 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,822 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/bb244f3e4da447618e9b67e7aab14b47, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fd1042c1befe40869cdae69db17d4d04, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/08ede8cb189340198b329c6181c90734, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/53706001c81540cb891df874f44d49ca] 2024-12-04T20:30:23,822 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f10d2d435a8423b8d91328fea5484d3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733344216503 2024-12-04T20:30:23,822 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb244f3e4da447618e9b67e7aab14b47, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733344216503 2024-12-04T20:30:23,822 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 79853bb3fe2d41c89425d8b830b0c8ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733344217664 2024-12-04T20:30:23,822 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1042c1befe40869cdae69db17d4d04, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733344217664 2024-12-04T20:30:23,823 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 585969f43ac743d6bb72556680621d93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733344218827 2024-12-04T20:30:23,823 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ede8cb189340198b329c6181c90734, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733344218827 2024-12-04T20:30:23,823 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3f1afd1b30443bafb6511bb0edd99f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733344221003 2024-12-04T20:30:23,823 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53706001c81540cb891df874f44d49ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733344221003 2024-12-04T20:30:23,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T20:30:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:23,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T20:30:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:23,830 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:23,838 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a7bcfb87fa1f45ad93ff7b8469e9475d_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344222150/Put/seqid=0 2024-12-04T20:30:23,839 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/9fe36b9620824932930bceac2c1cba33 is 50, key is test_row_0/B:col10/1733344221003/Put/seqid=0 2024-12-04T20:30:23,839 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204cec66085a8c54f54a3e897e5d8b5d468_eec55403d95e6b84b5f2b496767f2096 store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:23,842 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204cec66085a8c54f54a3e897e5d8b5d468_eec55403d95e6b84b5f2b496767f2096, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:23,842 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cec66085a8c54f54a3e897e5d8b5d468_eec55403d95e6b84b5f2b496767f2096 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:23,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742411_1587 (size=12561) 2024-12-04T20:30:23,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742412_1588 (size=4469) 2024-12-04T20:30:23,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742413_1589 (size=12304) 2024-12-04T20:30:24,251 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/9fe36b9620824932930bceac2c1cba33 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/9fe36b9620824932930bceac2c1cba33 2024-12-04T20:30:24,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:24,256 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:24,257 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/B of eec55403d95e6b84b5f2b496767f2096 into 9fe36b9620824932930bceac2c1cba33(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:24,257 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:24,257 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/B, priority=12, startTime=1733344223820; duration=0sec 2024-12-04T20:30:24,257 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:24,257 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:B 2024-12-04T20:30:24,257 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/2e5d3ff475e54409a4869b70d5f86749 is 175, key is test_row_0/A:col10/1733344221003/Put/seqid=0 2024-12-04T20:30:24,257 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:30:24,259 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:30:24,260 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/C is initiating minor compaction (all files) 2024-12-04T20:30:24,260 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/C in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:24,260 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9d5fa5c7ab434f718c4848470f2df6f2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0d0a1e978d3449290aef3930b245861, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8e52feed69ce4348a32dfb3a20dd194e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/23be68490be34b4887e4b6747276d4ca] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=47.7 K 2024-12-04T20:30:24,260 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d5fa5c7ab434f718c4848470f2df6f2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733344216503 2024-12-04T20:30:24,261 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f0d0a1e978d3449290aef3930b245861, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733344217664 2024-12-04T20:30:24,261 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e52feed69ce4348a32dfb3a20dd194e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733344218827 2024-12-04T20:30:24,261 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 23be68490be34b4887e4b6747276d4ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733344221003 2024-12-04T20:30:24,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a7bcfb87fa1f45ad93ff7b8469e9475d_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a7bcfb87fa1f45ad93ff7b8469e9475d_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fdcb254776374452978d6fba5a5e7e92, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:24,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742414_1590 (size=31515) 2024-12-04T20:30:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fdcb254776374452978d6fba5a5e7e92 is 175, key is test_row_0/A:col10/1733344222150/Put/seqid=0 2024-12-04T20:30:24,268 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/2e5d3ff475e54409a4869b70d5f86749 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2e5d3ff475e54409a4869b70d5f86749 2024-12-04T20:30:24,272 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/A of eec55403d95e6b84b5f2b496767f2096 into 2e5d3ff475e54409a4869b70d5f86749(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:24,272 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:24,273 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/A, priority=12, startTime=1733344223820; duration=0sec 2024-12-04T20:30:24,273 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:24,273 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:A 2024-12-04T20:30:24,279 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:24,280 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/87ee3c7555a942f6a5e7214532b58f2b is 50, key is test_row_0/C:col10/1733344221003/Put/seqid=0 2024-12-04T20:30:24,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742415_1591 (size=31105) 2024-12-04T20:30:24,289 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fdcb254776374452978d6fba5a5e7e92 2024-12-04T20:30:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/6cafe5f334db4a9da64782c66f59917d is 50, key is test_row_0/B:col10/1733344222150/Put/seqid=0 2024-12-04T20:30:24,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742416_1592 (size=12561) 2024-12-04T20:30:24,301 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/87ee3c7555a942f6a5e7214532b58f2b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/87ee3c7555a942f6a5e7214532b58f2b 2024-12-04T20:30:24,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742417_1593 (size=12151) 2024-12-04T20:30:24,305 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/6cafe5f334db4a9da64782c66f59917d 2024-12-04T20:30:24,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:24,317 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/C of eec55403d95e6b84b5f2b496767f2096 into 87ee3c7555a942f6a5e7214532b58f2b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:24,317 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:24,317 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/C, priority=12, startTime=1733344223821; duration=0sec 2024-12-04T20:30:24,317 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:24,317 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:C 2024-12-04T20:30:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/d4357fb8f6bd4c1890dcc52279da3ff8 is 50, key is test_row_0/C:col10/1733344222150/Put/seqid=0 2024-12-04T20:30:24,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742418_1594 (size=12151) 2024-12-04T20:30:24,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/d4357fb8f6bd4c1890dcc52279da3ff8 2024-12-04T20:30:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/fdcb254776374452978d6fba5a5e7e92 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fdcb254776374452978d6fba5a5e7e92 2024-12-04T20:30:24,334 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fdcb254776374452978d6fba5a5e7e92, entries=150, sequenceid=203, filesize=30.4 K 2024-12-04T20:30:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/6cafe5f334db4a9da64782c66f59917d as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6cafe5f334db4a9da64782c66f59917d 2024-12-04T20:30:24,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6cafe5f334db4a9da64782c66f59917d, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T20:30:24,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344284340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344284342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344284342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/d4357fb8f6bd4c1890dcc52279da3ff8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/d4357fb8f6bd4c1890dcc52279da3ff8 2024-12-04T20:30:24,355 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/d4357fb8f6bd4c1890dcc52279da3ff8, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T20:30:24,356 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for eec55403d95e6b84b5f2b496767f2096 in 529ms, sequenceid=203, compaction requested=false 2024-12-04T20:30:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-04T20:30:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-04T20:30:24,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-04T20:30:24,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-12-04T20:30:24,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.7590 sec 2024-12-04T20:30:24,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T20:30:24,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:24,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:24,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:24,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:24,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:24,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:24,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b13cfde5729c4373bb04e1b7d9f9355a_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344224452/Put/seqid=0 2024-12-04T20:30:24,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742419_1595 (size=12304) 2024-12-04T20:30:24,463 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:24,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b13cfde5729c4373bb04e1b7d9f9355a_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b13cfde5729c4373bb04e1b7d9f9355a_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:24,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/0e55b05d3e1844d3b7437ee7c5d74a76, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:24,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/0e55b05d3e1844d3b7437ee7c5d74a76 is 175, key is test_row_0/A:col10/1733344224452/Put/seqid=0 2024-12-04T20:30:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344284465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344284470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344284470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742420_1596 (size=31105) 2024-12-04T20:30:24,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/0e55b05d3e1844d3b7437ee7c5d74a76 2024-12-04T20:30:24,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/f56e16064dd04b16bbb9f603bfa61a3a is 50, key is test_row_0/B:col10/1733344224452/Put/seqid=0 2024-12-04T20:30:24,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742421_1597 (size=12151) 2024-12-04T20:30:24,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/f56e16064dd04b16bbb9f603bfa61a3a 2024-12-04T20:30:24,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/dab322fbc2434adc872181ec711fb9e8 is 50, key is test_row_0/C:col10/1733344224452/Put/seqid=0 2024-12-04T20:30:24,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742422_1598 (size=12151) 2024-12-04T20:30:24,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/dab322fbc2434adc872181ec711fb9e8 2024-12-04T20:30:24,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/0e55b05d3e1844d3b7437ee7c5d74a76 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/0e55b05d3e1844d3b7437ee7c5d74a76 2024-12-04T20:30:24,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/0e55b05d3e1844d3b7437ee7c5d74a76, entries=150, sequenceid=227, filesize=30.4 K 2024-12-04T20:30:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/f56e16064dd04b16bbb9f603bfa61a3a as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/f56e16064dd04b16bbb9f603bfa61a3a 2024-12-04T20:30:24,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/f56e16064dd04b16bbb9f603bfa61a3a, entries=150, sequenceid=227, filesize=11.9 K 2024-12-04T20:30:24,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/dab322fbc2434adc872181ec711fb9e8 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/dab322fbc2434adc872181ec711fb9e8 2024-12-04T20:30:24,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/dab322fbc2434adc872181ec711fb9e8, entries=150, sequenceid=227, filesize=11.9 K 2024-12-04T20:30:24,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for eec55403d95e6b84b5f2b496767f2096 in 92ms, sequenceid=227, compaction requested=true 2024-12-04T20:30:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:30:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:24,545 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:30:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:24,545 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:30:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:24,547 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:24,547 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:24,547 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/A is initiating minor compaction (all files) 2024-12-04T20:30:24,547 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/B is initiating minor compaction (all files) 2024-12-04T20:30:24,547 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/A in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:24,547 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/B in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:24,547 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/9fe36b9620824932930bceac2c1cba33, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6cafe5f334db4a9da64782c66f59917d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/f56e16064dd04b16bbb9f603bfa61a3a] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=36.0 K 2024-12-04T20:30:24,547 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2e5d3ff475e54409a4869b70d5f86749, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fdcb254776374452978d6fba5a5e7e92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/0e55b05d3e1844d3b7437ee7c5d74a76] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=91.5 K 2024-12-04T20:30:24,547 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:24,547 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2e5d3ff475e54409a4869b70d5f86749, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fdcb254776374452978d6fba5a5e7e92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/0e55b05d3e1844d3b7437ee7c5d74a76] 2024-12-04T20:30:24,548 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe36b9620824932930bceac2c1cba33, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733344221003 2024-12-04T20:30:24,548 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e5d3ff475e54409a4869b70d5f86749, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733344221003 2024-12-04T20:30:24,549 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cafe5f334db4a9da64782c66f59917d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733344222150 2024-12-04T20:30:24,549 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdcb254776374452978d6fba5a5e7e92, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733344222150 2024-12-04T20:30:24,549 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e55b05d3e1844d3b7437ee7c5d74a76, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733344224327 2024-12-04T20:30:24,550 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f56e16064dd04b16bbb9f603bfa61a3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733344224327 2024-12-04T20:30:24,558 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:24,560 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:24,561 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/21ca4b3ded8d4549a7eee0ef6d325c09 is 50, key is test_row_0/B:col10/1733344224452/Put/seqid=0 2024-12-04T20:30:24,578 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120478cbdb50b78248eebce0fad6052310dd_eec55403d95e6b84b5f2b496767f2096 store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:24,579 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120478cbdb50b78248eebce0fad6052310dd_eec55403d95e6b84b5f2b496767f2096, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:24,580 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120478cbdb50b78248eebce0fad6052310dd_eec55403d95e6b84b5f2b496767f2096 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:24,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T20:30:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:24,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742423_1599 (size=12663) 2024-12-04T20:30:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742424_1600 (size=4469) 2024-12-04T20:30:24,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042494b0f03dff4ffa97686141bca0072c_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344224585/Put/seqid=0 2024-12-04T20:30:24,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344284617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344284615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742425_1601 (size=17284) 2024-12-04T20:30:24,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344284621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:24,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042494b0f03dff4ffa97686141bca0072c_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042494b0f03dff4ffa97686141bca0072c_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:24,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ff454c85aaff465eaa961ef1e8e5629b, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:24,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ff454c85aaff465eaa961ef1e8e5629b is 175, key is test_row_0/A:col10/1733344224585/Put/seqid=0 2024-12-04T20:30:24,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742426_1602 (size=48389) 2024-12-04T20:30:24,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ff454c85aaff465eaa961ef1e8e5629b 2024-12-04T20:30:24,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/425467c604dc46368c12d00f56f2ec93 is 50, key is test_row_0/B:col10/1733344224585/Put/seqid=0 2024-12-04T20:30:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742427_1603 (size=12151) 2024-12-04T20:30:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344284731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344284731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344284731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T20:30:24,752 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:24,753 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-04T20:30:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T20:30:24,755 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:24,755 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:24,756 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T20:30:24,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T20:30:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:24,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:24,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344284941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344284942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344284942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,010 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/21ca4b3ded8d4549a7eee0ef6d325c09 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/21ca4b3ded8d4549a7eee0ef6d325c09 2024-12-04T20:30:25,015 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/B of eec55403d95e6b84b5f2b496767f2096 into 21ca4b3ded8d4549a7eee0ef6d325c09(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:25,015 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:25,015 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/B, priority=13, startTime=1733344224545; duration=0sec 2024-12-04T20:30:25,016 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:25,016 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:B 2024-12-04T20:30:25,016 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:25,017 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:25,017 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/C is initiating minor compaction (all files) 2024-12-04T20:30:25,017 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/C in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,018 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/87ee3c7555a942f6a5e7214532b58f2b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/d4357fb8f6bd4c1890dcc52279da3ff8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/dab322fbc2434adc872181ec711fb9e8] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=36.0 K 2024-12-04T20:30:25,018 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#A#compaction#501 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:25,018 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ee3c7555a942f6a5e7214532b58f2b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733344221003 2024-12-04T20:30:25,018 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting d4357fb8f6bd4c1890dcc52279da3ff8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733344222150 2024-12-04T20:30:25,018 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/2d9c6874949d491ca4347a63b181bd5f is 175, key is test_row_0/A:col10/1733344224452/Put/seqid=0 2024-12-04T20:30:25,019 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting dab322fbc2434adc872181ec711fb9e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733344224327 2024-12-04T20:30:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742428_1604 (size=31617) 2024-12-04T20:30:25,027 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#C#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:25,027 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/f0354a5fae3544aba253f102653ac1f3 is 50, key is test_row_0/C:col10/1733344224452/Put/seqid=0 2024-12-04T20:30:25,030 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/2d9c6874949d491ca4347a63b181bd5f as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2d9c6874949d491ca4347a63b181bd5f 2024-12-04T20:30:25,045 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/A of eec55403d95e6b84b5f2b496767f2096 into 2d9c6874949d491ca4347a63b181bd5f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:25,045 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:25,045 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/A, priority=13, startTime=1733344224545; duration=0sec 2024-12-04T20:30:25,045 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:25,045 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:A 2024-12-04T20:30:25,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742429_1605 (size=12663) 2024-12-04T20:30:25,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/425467c604dc46368c12d00f56f2ec93 2024-12-04T20:30:25,058 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/f0354a5fae3544aba253f102653ac1f3 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0354a5fae3544aba253f102653ac1f3 2024-12-04T20:30:25,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T20:30:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/6060b720bfd344fdbb8148db639754fe is 50, key is test_row_0/C:col10/1733344224585/Put/seqid=0 2024-12-04T20:30:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,066 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/C of eec55403d95e6b84b5f2b496767f2096 into f0354a5fae3544aba253f102653ac1f3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:25,066 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:25,066 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/C, priority=13, startTime=1733344224545; duration=0sec 2024-12-04T20:30:25,066 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:25,066 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:C 2024-12-04T20:30:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T20:30:25,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742430_1606 (size=12151) 2024-12-04T20:30:25,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T20:30:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344285251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344285251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344285251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T20:30:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T20:30:25,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/6060b720bfd344fdbb8148db639754fe 2024-12-04T20:30:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ff454c85aaff465eaa961ef1e8e5629b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ff454c85aaff465eaa961ef1e8e5629b 2024-12-04T20:30:25,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ff454c85aaff465eaa961ef1e8e5629b, entries=250, sequenceid=242, filesize=47.3 K 2024-12-04T20:30:25,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/425467c604dc46368c12d00f56f2ec93 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/425467c604dc46368c12d00f56f2ec93 2024-12-04T20:30:25,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/425467c604dc46368c12d00f56f2ec93, entries=150, sequenceid=242, filesize=11.9 K 2024-12-04T20:30:25,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/6060b720bfd344fdbb8148db639754fe as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6060b720bfd344fdbb8148db639754fe 2024-12-04T20:30:25,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6060b720bfd344fdbb8148db639754fe, entries=150, sequenceid=242, filesize=11.9 K 2024-12-04T20:30:25,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for eec55403d95e6b84b5f2b496767f2096 in 919ms, sequenceid=242, compaction requested=false 2024-12-04T20:30:25,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:25,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T20:30:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:25,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T20:30:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:25,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d6000b1b218043c9a4f94669a47da364_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344224613/Put/seqid=0 2024-12-04T20:30:25,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742431_1607 (size=12404) 2024-12-04T20:30:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:25,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344285778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344285779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344285782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T20:30:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344285892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344285892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344285892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:25,959 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d6000b1b218043c9a4f94669a47da364_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d6000b1b218043c9a4f94669a47da364_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/97f00470c61c407097cdf2b8173ad7c2, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/97f00470c61c407097cdf2b8173ad7c2 is 175, key is test_row_0/A:col10/1733344224613/Put/seqid=0 2024-12-04T20:30:25,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742432_1608 (size=31205) 2024-12-04T20:30:25,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/97f00470c61c407097cdf2b8173ad7c2 2024-12-04T20:30:25,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/5bfe56de6dc14775a7086178a929b36c is 50, key is test_row_0/B:col10/1733344224613/Put/seqid=0 2024-12-04T20:30:25,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742433_1609 (size=12251) 2024-12-04T20:30:25,995 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/5bfe56de6dc14775a7086178a929b36c 2024-12-04T20:30:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/c766a2b611dd41f398f43dca57335096 is 50, key is test_row_0/C:col10/1733344224613/Put/seqid=0 2024-12-04T20:30:26,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742434_1610 (size=12251) 2024-12-04T20:30:26,020 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/c766a2b611dd41f398f43dca57335096 2024-12-04T20:30:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/97f00470c61c407097cdf2b8173ad7c2 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/97f00470c61c407097cdf2b8173ad7c2 2024-12-04T20:30:26,036 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/97f00470c61c407097cdf2b8173ad7c2, entries=150, sequenceid=262, filesize=30.5 K 2024-12-04T20:30:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/5bfe56de6dc14775a7086178a929b36c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5bfe56de6dc14775a7086178a929b36c 2024-12-04T20:30:26,039 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5bfe56de6dc14775a7086178a929b36c, entries=150, sequenceid=262, filesize=12.0 K 2024-12-04T20:30:26,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/c766a2b611dd41f398f43dca57335096 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c766a2b611dd41f398f43dca57335096 2024-12-04T20:30:26,043 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c766a2b611dd41f398f43dca57335096, entries=150, sequenceid=262, filesize=12.0 K 2024-12-04T20:30:26,044 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for eec55403d95e6b84b5f2b496767f2096 in 524ms, sequenceid=262, compaction requested=true 2024-12-04T20:30:26,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:26,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:26,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-04T20:30:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-04T20:30:26,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-04T20:30:26,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-12-04T20:30:26,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.2940 sec 2024-12-04T20:30:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:26,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T20:30:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:26,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b8a4bedb3330458e86d9f8899ad12925_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344226103/Put/seqid=0 2024-12-04T20:30:26,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742435_1611 (size=17534) 2024-12-04T20:30:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344286127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344286134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344286134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344286241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344286241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344286241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344286451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344286451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344286451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,529 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:26,532 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b8a4bedb3330458e86d9f8899ad12925_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b8a4bedb3330458e86d9f8899ad12925_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:26,533 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ba1cf72bd84c473dab33da6dca0acbe0, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:26,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ba1cf72bd84c473dab33da6dca0acbe0 is 175, key is test_row_0/A:col10/1733344226103/Put/seqid=0 2024-12-04T20:30:26,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742436_1612 (size=48639) 2024-12-04T20:30:26,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344286761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344286761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344286761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T20:30:26,902 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:26,902 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T20:30:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-04T20:30:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T20:30:26,904 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:30:26,904 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:30:26,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:30:26,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ba1cf72bd84c473dab33da6dca0acbe0 2024-12-04T20:30:26,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/7784053185984c9d96ee5a6af04f6900 is 50, key is test_row_0/B:col10/1733344226103/Put/seqid=0 2024-12-04T20:30:26,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742437_1613 (size=12301) 2024-12-04T20:30:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T20:30:27,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43116 deadline: 1733344287042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:27,045 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_0' in eec55403d95e6b84b5f2b496767f2096 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:30:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:27,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T20:30:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43134 deadline: 1733344287061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:27,064 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2ef38372881,45443,1733344054291 for 'test_row_2' in eec55403d95e6b84b5f2b496767f2096 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T20:30:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:27,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T20:30:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T20:30:27,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344287271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:27,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344287271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:27,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344287271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:27,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/7784053185984c9d96ee5a6af04f6900 2024-12-04T20:30:27,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/59502ecb5e274126a788650af288d95b is 50, key is test_row_0/C:col10/1733344226103/Put/seqid=0 2024-12-04T20:30:27,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742438_1614 (size=12301) 2024-12-04T20:30:27,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T20:30:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T20:30:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T20:30:27,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T20:30:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:30:27,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/59502ecb5e274126a788650af288d95b 2024-12-04T20:30:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/ba1cf72bd84c473dab33da6dca0acbe0 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ba1cf72bd84c473dab33da6dca0acbe0 2024-12-04T20:30:27,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ba1cf72bd84c473dab33da6dca0acbe0, entries=250, sequenceid=280, filesize=47.5 K 2024-12-04T20:30:27,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/7784053185984c9d96ee5a6af04f6900 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7784053185984c9d96ee5a6af04f6900 2024-12-04T20:30:27,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7784053185984c9d96ee5a6af04f6900, entries=150, sequenceid=280, filesize=12.0 K 2024-12-04T20:30:27,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/59502ecb5e274126a788650af288d95b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/59502ecb5e274126a788650af288d95b 2024-12-04T20:30:27,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/59502ecb5e274126a788650af288d95b, entries=150, sequenceid=280, filesize=12.0 K 2024-12-04T20:30:27,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for eec55403d95e6b84b5f2b496767f2096 in 1680ms, sequenceid=280, compaction requested=true 2024-12-04T20:30:27,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:27,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:30:27,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:27,785 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:30:27,785 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:30:27,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:30:27,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:27,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:30:27,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:27,793 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:30:27,793 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:30:27,793 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/B is initiating minor compaction (all files) 2024-12-04T20:30:27,793 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/A is initiating minor compaction (all files) 2024-12-04T20:30:27,793 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/B in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,793 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/21ca4b3ded8d4549a7eee0ef6d325c09, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/425467c604dc46368c12d00f56f2ec93, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5bfe56de6dc14775a7086178a929b36c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7784053185984c9d96ee5a6af04f6900] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=48.2 K 2024-12-04T20:30:27,794 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/A in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,794 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2d9c6874949d491ca4347a63b181bd5f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ff454c85aaff465eaa961ef1e8e5629b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/97f00470c61c407097cdf2b8173ad7c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ba1cf72bd84c473dab33da6dca0acbe0] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=156.1 K 2024-12-04T20:30:27,794 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,794 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2d9c6874949d491ca4347a63b181bd5f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ff454c85aaff465eaa961ef1e8e5629b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/97f00470c61c407097cdf2b8173ad7c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ba1cf72bd84c473dab33da6dca0acbe0] 2024-12-04T20:30:27,794 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ca4b3ded8d4549a7eee0ef6d325c09, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733344224327 2024-12-04T20:30:27,795 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 425467c604dc46368c12d00f56f2ec93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733344224585 2024-12-04T20:30:27,795 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d9c6874949d491ca4347a63b181bd5f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733344224327 2024-12-04T20:30:27,795 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff454c85aaff465eaa961ef1e8e5629b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733344224461 2024-12-04T20:30:27,795 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bfe56de6dc14775a7086178a929b36c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733344224613 2024-12-04T20:30:27,795 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f00470c61c407097cdf2b8173ad7c2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733344224613 2024-12-04T20:30:27,795 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 7784053185984c9d96ee5a6af04f6900, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733344225776 2024-12-04T20:30:27,796 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1cf72bd84c473dab33da6dca0acbe0, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733344225776 2024-12-04T20:30:27,806 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:27,807 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/016d8e615f7843c3975d4ce87987fcac is 50, key is test_row_0/B:col10/1733344226103/Put/seqid=0 2024-12-04T20:30:27,814 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:27,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45443 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T20:30:27,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:27,822 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T20:30:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:27,836 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204888b7c2e13ea4a6897891ae043526606_eec55403d95e6b84b5f2b496767f2096 store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:27,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742439_1615 (size=12949) 2024-12-04T20:30:27,838 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204888b7c2e13ea4a6897891ae043526606_eec55403d95e6b84b5f2b496767f2096, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:27,838 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204888b7c2e13ea4a6897891ae043526606_eec55403d95e6b84b5f2b496767f2096 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:27,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045cf4bad543e24d698d83a627649727b2_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344226131/Put/seqid=0 2024-12-04T20:30:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742440_1616 (size=4469) 2024-12-04T20:30:27,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742441_1617 (size=12454) 2024-12-04T20:30:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:27,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045cf4bad543e24d698d83a627649727b2_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045cf4bad543e24d698d83a627649727b2_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/68dc8993850a4cb0983f8c45022fdf81, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:27,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/68dc8993850a4cb0983f8c45022fdf81 is 175, key is test_row_0/A:col10/1733344226131/Put/seqid=0 2024-12-04T20:30:27,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742442_1618 (size=31255) 2024-12-04T20:30:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T20:30:28,241 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/016d8e615f7843c3975d4ce87987fcac as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/016d8e615f7843c3975d4ce87987fcac 2024-12-04T20:30:28,244 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/B of eec55403d95e6b84b5f2b496767f2096 into 016d8e615f7843c3975d4ce87987fcac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:28,244 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:28,244 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/B, priority=12, startTime=1733344227785; duration=0sec 2024-12-04T20:30:28,245 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:28,245 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:B 2024-12-04T20:30:28,245 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:30:28,246 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T20:30:28,246 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/C is initiating minor compaction (all files) 2024-12-04T20:30:28,246 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/C in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:28,246 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0354a5fae3544aba253f102653ac1f3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6060b720bfd344fdbb8148db639754fe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c766a2b611dd41f398f43dca57335096, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/59502ecb5e274126a788650af288d95b] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=48.2 K 2024-12-04T20:30:28,246 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting f0354a5fae3544aba253f102653ac1f3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733344224327 2024-12-04T20:30:28,246 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 6060b720bfd344fdbb8148db639754fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733344224585 2024-12-04T20:30:28,247 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting c766a2b611dd41f398f43dca57335096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733344224613 2024-12-04T20:30:28,247 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 59502ecb5e274126a788650af288d95b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733344225776 2024-12-04T20:30:28,248 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:28,248 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/c42125b97c0c49f8b899aa9c82889b5c is 175, key is test_row_0/A:col10/1733344226103/Put/seqid=0 2024-12-04T20:30:28,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742443_1619 (size=31903) 2024-12-04T20:30:28,255 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:28,256 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/0641819ea9f14d379e811c6b9296d1ec is 50, key is test_row_0/C:col10/1733344226103/Put/seqid=0 2024-12-04T20:30:28,256 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/c42125b97c0c49f8b899aa9c82889b5c as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c42125b97c0c49f8b899aa9c82889b5c 2024-12-04T20:30:28,260 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/A of eec55403d95e6b84b5f2b496767f2096 into c42125b97c0c49f8b899aa9c82889b5c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:28,260 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:28,261 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/A, priority=12, startTime=1733344227785; duration=0sec 2024-12-04T20:30:28,261 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:28,261 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:A 2024-12-04T20:30:28,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/68dc8993850a4cb0983f8c45022fdf81 2024-12-04T20:30:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742444_1620 (size=12949) 2024-12-04T20:30:28,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/51cf81af2a1c454790354c122ea8de27 is 50, key is test_row_0/B:col10/1733344226131/Put/seqid=0 2024-12-04T20:30:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. as already flushing 2024-12-04T20:30:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:28,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742445_1621 (size=12301) 2024-12-04T20:30:28,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344288309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344288309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344288310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344288421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344288421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344288421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344288632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344288632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344288632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,669 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/0641819ea9f14d379e811c6b9296d1ec as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/0641819ea9f14d379e811c6b9296d1ec 2024-12-04T20:30:28,673 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/C of eec55403d95e6b84b5f2b496767f2096 into 0641819ea9f14d379e811c6b9296d1ec(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:28,673 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:28,673 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/C, priority=12, startTime=1733344227785; duration=0sec 2024-12-04T20:30:28,673 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:28,673 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:C 2024-12-04T20:30:28,695 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/51cf81af2a1c454790354c122ea8de27 2024-12-04T20:30:28,699 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-12-04T20:30:28,699 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:28,699 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,699 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:28,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/08f770a0896b4fc0b9d379f7c9a86d04 is 50, key is test_row_0/C:col10/1733344226131/Put/seqid=0 2024-12-04T20:30:28,700 INFO [Thread-2419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2419. 2024-12-04T20:30:28,700 DEBUG [Thread-2419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:28,700 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,700 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:28,701 INFO [Thread-2413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2413. 2024-12-04T20:30:28,701 DEBUG [Thread-2413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:28,701 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,702 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:28,702 INFO [Thread-2417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2417. 2024-12-04T20:30:28,702 DEBUG [Thread-2417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:28,702 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,702 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:28,703 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-12-04T20:30:28,703 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:28,703 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,703 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:28,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:28,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742446_1622 (size=12301) 2024-12-04T20:30:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:30:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43158 deadline: 1733344288942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733344288942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43132 deadline: 1733344288942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 2024-12-04T20:30:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 , the old value is region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:30:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., hostname=c2ef38372881,45443,1733344054291, seqNum=5 because the exception is null or not the one we care about 2024-12-04T20:30:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T20:30:29,104 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/08f770a0896b4fc0b9d379f7c9a86d04 2024-12-04T20:30:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/68dc8993850a4cb0983f8c45022fdf81 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/68dc8993850a4cb0983f8c45022fdf81 2024-12-04T20:30:29,111 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/68dc8993850a4cb0983f8c45022fdf81, entries=150, sequenceid=294, filesize=30.5 K 2024-12-04T20:30:29,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/51cf81af2a1c454790354c122ea8de27 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/51cf81af2a1c454790354c122ea8de27 2024-12-04T20:30:29,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/51cf81af2a1c454790354c122ea8de27, entries=150, sequenceid=294, filesize=12.0 K 2024-12-04T20:30:29,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/08f770a0896b4fc0b9d379f7c9a86d04 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/08f770a0896b4fc0b9d379f7c9a86d04 2024-12-04T20:30:29,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/08f770a0896b4fc0b9d379f7c9a86d04, entries=150, sequenceid=294, filesize=12.0 K 2024-12-04T20:30:29,118 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for eec55403d95e6b84b5f2b496767f2096 in 1297ms, sequenceid=294, compaction requested=false 2024-12-04T20:30:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-04T20:30:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-04T20:30:29,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-04T20:30:29,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2150 sec 2024-12-04T20:30:29,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.2190 sec 2024-12-04T20:30:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45443 {}] regionserver.HRegion(8855): Flush requested on eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:29,454 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-12-04T20:30:29,454 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:29,454 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:29,454 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:29,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:29,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733344228307, max timestamp=1733344229452Num uniques 859; 2024-12-04T20:30:29,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733344228307, max timestamp=1733344229452Num uniques 859; 2024-12-04T20:30:29,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=10116, suffix heap size=117064,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=221 2024-12-04T20:30:29,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:30:29,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733344228307, max timestamp=1733344229452Num uniques 859; 2024-12-04T20:30:29,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c2ef38372881:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T20:30:29,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=38.56 KB heapSize=92.84 KB 2024-12-04T20:30:29,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:29,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:29,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:29,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:29,456 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-12-04T20:30:29,456 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-12-04T20:30:29,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:29,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T20:30:29,456 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:29,456 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:29,456 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:29,456 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:29,456 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:29,456 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:29,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:29,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:29,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120466149cfadee74b09bbac38d8bda405c5_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344229452/Put/seqid=0 2024-12-04T20:30:29,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742447_1623 (size=12454) 2024-12-04T20:30:29,866 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:29,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120466149cfadee74b09bbac38d8bda405c5_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120466149cfadee74b09bbac38d8bda405c5_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:29,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/59ebba31bc4346968fc196e5d05a55e9, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:29,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/59ebba31bc4346968fc196e5d05a55e9 is 175, key is test_row_0/A:col10/1733344229452/Put/seqid=0 2024-12-04T20:30:29,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742448_1624 (size=31255) 2024-12-04T20:30:30,280 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/59ebba31bc4346968fc196e5d05a55e9 2024-12-04T20:30:30,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/bf474f96f32c415fbefa01d9d5c77759 is 50, key is test_row_0/B:col10/1733344229452/Put/seqid=0 2024-12-04T20:30:30,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742449_1625 (size=12301) 2024-12-04T20:30:30,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/bf474f96f32c415fbefa01d9d5c77759 2024-12-04T20:30:30,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/3ee2f34ffbb540aba05d2bcbc8010205 is 50, key is test_row_0/C:col10/1733344229452/Put/seqid=0 2024-12-04T20:30:30,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742450_1626 (size=12301) 2024-12-04T20:30:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T20:30:31,082 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:31,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.72 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/3ee2f34ffbb540aba05d2bcbc8010205 2024-12-04T20:30:31,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/59ebba31bc4346968fc196e5d05a55e9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/59ebba31bc4346968fc196e5d05a55e9 2024-12-04T20:30:31,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/59ebba31bc4346968fc196e5d05a55e9, entries=150, sequenceid=318, filesize=30.5 K 2024-12-04T20:30:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/bf474f96f32c415fbefa01d9d5c77759 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf474f96f32c415fbefa01d9d5c77759 2024-12-04T20:30:31,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf474f96f32c415fbefa01d9d5c77759, entries=150, sequenceid=318, filesize=12.0 K 2024-12-04T20:30:31,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/3ee2f34ffbb540aba05d2bcbc8010205 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/3ee2f34ffbb540aba05d2bcbc8010205 2024-12-04T20:30:31,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/3ee2f34ffbb540aba05d2bcbc8010205, entries=150, sequenceid=318, filesize=12.0 K 2024-12-04T20:30:31,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~38.56 KB/39482, heapSize ~92.80 KB/95024, currentSize=0 B/0 for eec55403d95e6b84b5f2b496767f2096 in 1662ms, sequenceid=318, compaction requested=true 2024-12-04T20:30:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:30:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:31,117 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:30:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:31,117 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eec55403d95e6b84b5f2b496767f2096:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T20:30:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:31,118 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:31,118 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:31,118 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/A is initiating minor compaction (all files) 2024-12-04T20:30:31,118 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/B is initiating minor compaction (all files) 2024-12-04T20:30:31,118 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/B in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:31,118 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/A in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:31,118 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c42125b97c0c49f8b899aa9c82889b5c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/68dc8993850a4cb0983f8c45022fdf81, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/59ebba31bc4346968fc196e5d05a55e9] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=92.2 K 2024-12-04T20:30:31,118 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/016d8e615f7843c3975d4ce87987fcac, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/51cf81af2a1c454790354c122ea8de27, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf474f96f32c415fbefa01d9d5c77759] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=36.7 K 2024-12-04T20:30:31,118 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:31,118 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. files: [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c42125b97c0c49f8b899aa9c82889b5c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/68dc8993850a4cb0983f8c45022fdf81, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/59ebba31bc4346968fc196e5d05a55e9] 2024-12-04T20:30:31,118 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 016d8e615f7843c3975d4ce87987fcac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733344225776 2024-12-04T20:30:31,118 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42125b97c0c49f8b899aa9c82889b5c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733344225776 2024-12-04T20:30:31,119 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 51cf81af2a1c454790354c122ea8de27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733344226125 2024-12-04T20:30:31,119 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68dc8993850a4cb0983f8c45022fdf81, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733344226125 2024-12-04T20:30:31,119 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting bf474f96f32c415fbefa01d9d5c77759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733344228308 2024-12-04T20:30:31,119 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ebba31bc4346968fc196e5d05a55e9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733344228308 2024-12-04T20:30:31,123 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:31,125 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120453627ecab08a48feb8ab034fadaebba7_eec55403d95e6b84b5f2b496767f2096 store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:31,126 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#B#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:31,126 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/5c510117a1a54eb59ec685f9c1514035 is 50, key is test_row_0/B:col10/1733344229452/Put/seqid=0 2024-12-04T20:30:31,128 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120453627ecab08a48feb8ab034fadaebba7_eec55403d95e6b84b5f2b496767f2096, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:31,128 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120453627ecab08a48feb8ab034fadaebba7_eec55403d95e6b84b5f2b496767f2096 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:31,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742451_1627 (size=13051) 2024-12-04T20:30:31,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742452_1628 (size=4469) 2024-12-04T20:30:31,539 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/5c510117a1a54eb59ec685f9c1514035 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5c510117a1a54eb59ec685f9c1514035 2024-12-04T20:30:31,539 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:31,540 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/c9525d709d7e487b8eaa0dce429ef115 is 175, key is test_row_0/A:col10/1733344229452/Put/seqid=0 2024-12-04T20:30:31,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742453_1629 (size=32005) 2024-12-04T20:30:31,545 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/B of eec55403d95e6b84b5f2b496767f2096 into 5c510117a1a54eb59ec685f9c1514035(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:31,545 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:31,545 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/B, priority=13, startTime=1733344231117; duration=0sec 2024-12-04T20:30:31,545 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T20:30:31,545 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:B 2024-12-04T20:30:31,545 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:30:31,546 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:30:31,546 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1541): eec55403d95e6b84b5f2b496767f2096/C is initiating minor compaction (all files) 2024-12-04T20:30:31,547 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eec55403d95e6b84b5f2b496767f2096/C in TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:31,547 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/0641819ea9f14d379e811c6b9296d1ec, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/08f770a0896b4fc0b9d379f7c9a86d04, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/3ee2f34ffbb540aba05d2bcbc8010205] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp, totalSize=36.7 K 2024-12-04T20:30:31,547 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 0641819ea9f14d379e811c6b9296d1ec, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733344225776 2024-12-04T20:30:31,547 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 08f770a0896b4fc0b9d379f7c9a86d04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733344226125 2024-12-04T20:30:31,548 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee2f34ffbb540aba05d2bcbc8010205, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733344228308 2024-12-04T20:30:31,556 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eec55403d95e6b84b5f2b496767f2096#C#compaction#524 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:30:31,556 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/6fa109cc39484809a5e60602d91cf466 is 50, key is test_row_0/C:col10/1733344229452/Put/seqid=0 2024-12-04T20:30:31,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742454_1630 (size=13051) 2024-12-04T20:30:31,950 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/c9525d709d7e487b8eaa0dce429ef115 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c9525d709d7e487b8eaa0dce429ef115 2024-12-04T20:30:31,956 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/A of eec55403d95e6b84b5f2b496767f2096 into c9525d709d7e487b8eaa0dce429ef115(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:31,956 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:31,956 INFO [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/A, priority=13, startTime=1733344231117; duration=0sec 2024-12-04T20:30:31,957 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:31,957 DEBUG [RS:0;c2ef38372881:45443-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:A 2024-12-04T20:30:31,967 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/6fa109cc39484809a5e60602d91cf466 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6fa109cc39484809a5e60602d91cf466 2024-12-04T20:30:31,974 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eec55403d95e6b84b5f2b496767f2096/C of eec55403d95e6b84b5f2b496767f2096 into 6fa109cc39484809a5e60602d91cf466(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:30:31,974 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eec55403d95e6b84b5f2b496767f2096: 2024-12-04T20:30:31,975 INFO [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., storeName=eec55403d95e6b84b5f2b496767f2096/C, priority=13, startTime=1733344231117; duration=0sec 2024-12-04T20:30:31,975 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:30:31,975 DEBUG [RS:0;c2ef38372881:45443-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eec55403d95e6b84b5f2b496767f2096:C 2024-12-04T20:30:32,787 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:30:37,113 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-12-04T20:30:37,113 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:37,113 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:37,113 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:37,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:37,165 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-12-04T20:30:37,165 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T20:30:37,165 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:37,165 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:37,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4255 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3956 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4099 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4229 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3966 2024-12-04T20:30:37,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T20:30:37,166 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:30:37,166 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:30:37,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:37,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:37,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:37,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T20:30:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T20:30:37,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344237171"}]},"ts":"1733344237171"} 2024-12-04T20:30:37,173 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T20:30:37,173 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T20:30:37,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T20:30:37,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, UNASSIGN}] 2024-12-04T20:30:37,175 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, UNASSIGN 2024-12-04T20:30:37,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=CLOSING, regionLocation=c2ef38372881,45443,1733344054291 2024-12-04T20:30:37,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, UNASSIGN because future has completed 2024-12-04T20:30:37,177 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:30:37,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291}] 2024-12-04T20:30:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T20:30:37,329 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:37,329 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:30:37,329 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing eec55403d95e6b84b5f2b496767f2096, disabling compactions & flushes 2024-12-04T20:30:37,329 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:37,329 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:37,329 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. after waiting 0 ms 2024-12-04T20:30:37,329 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:37,329 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing eec55403d95e6b84b5f2b496767f2096 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-04T20:30:37,330 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=A 2024-12-04T20:30:37,330 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:37,330 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=B 2024-12-04T20:30:37,330 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:37,330 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eec55403d95e6b84b5f2b496767f2096, store=C 2024-12-04T20:30:37,330 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T20:30:37,334 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042f8a30190a9d440e85af70d85a4bff5d_eec55403d95e6b84b5f2b496767f2096 is 50, key is test_row_0/A:col10/1733344237112/Put/seqid=0 2024-12-04T20:30:37,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742455_1631 (size=9914) 2024-12-04T20:30:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T20:30:37,738 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:30:37,747 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042f8a30190a9d440e85af70d85a4bff5d_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042f8a30190a9d440e85af70d85a4bff5d_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:37,748 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/4b912415444542ecab3f869595390330, store: [table=TestAcidGuarantees family=A region=eec55403d95e6b84b5f2b496767f2096] 2024-12-04T20:30:37,749 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/4b912415444542ecab3f869595390330 is 175, key is test_row_0/A:col10/1733344237112/Put/seqid=0 2024-12-04T20:30:37,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742456_1632 (size=22561) 2024-12-04T20:30:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T20:30:38,153 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/4b912415444542ecab3f869595390330 2024-12-04T20:30:38,158 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/c3c7635022e6463f9fc842e38d0f6301 is 50, key is test_row_0/B:col10/1733344237112/Put/seqid=0 2024-12-04T20:30:38,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742457_1633 (size=9857) 2024-12-04T20:30:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T20:30:38,564 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/c3c7635022e6463f9fc842e38d0f6301 2024-12-04T20:30:38,568 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/aa50b80899e9424b952f741a2bb10a34 is 50, key is test_row_0/C:col10/1733344237112/Put/seqid=0 2024-12-04T20:30:38,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742458_1634 (size=9857) 2024-12-04T20:30:38,972 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/aa50b80899e9424b952f741a2bb10a34 2024-12-04T20:30:38,975 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/A/4b912415444542ecab3f869595390330 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/4b912415444542ecab3f869595390330 2024-12-04T20:30:38,978 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/4b912415444542ecab3f869595390330, entries=100, sequenceid=326, filesize=22.0 K 2024-12-04T20:30:38,978 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/B/c3c7635022e6463f9fc842e38d0f6301 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/c3c7635022e6463f9fc842e38d0f6301 2024-12-04T20:30:38,981 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/c3c7635022e6463f9fc842e38d0f6301, entries=100, sequenceid=326, filesize=9.6 K 2024-12-04T20:30:38,982 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/.tmp/C/aa50b80899e9424b952f741a2bb10a34 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/aa50b80899e9424b952f741a2bb10a34 2024-12-04T20:30:38,984 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/aa50b80899e9424b952f741a2bb10a34, entries=100, sequenceid=326, filesize=9.6 K 2024-12-04T20:30:38,985 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for eec55403d95e6b84b5f2b496767f2096 in 1656ms, sequenceid=326, compaction requested=false 2024-12-04T20:30:38,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a34704895726432d987ffb4cea4436b9, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/47c830875d474bc48f53e7db022ae662, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/f6dbfa704f724cc8afad636010db8ea7, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/beb02c381e214d9d8f6014360be16e33, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a670be7bc72048f6975f103686d74599, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/67f2dc3668e94e16bc19978e580323e6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/77ddeb6d392d473187120fa5e4f0ca19, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/92b4dee409af4ef886e62921895c2ffe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/bb244f3e4da447618e9b67e7aab14b47, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fd1042c1befe40869cdae69db17d4d04, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/08ede8cb189340198b329c6181c90734, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/53706001c81540cb891df874f44d49ca, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2e5d3ff475e54409a4869b70d5f86749, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fdcb254776374452978d6fba5a5e7e92, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2d9c6874949d491ca4347a63b181bd5f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/0e55b05d3e1844d3b7437ee7c5d74a76, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ff454c85aaff465eaa961ef1e8e5629b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/97f00470c61c407097cdf2b8173ad7c2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ba1cf72bd84c473dab33da6dca0acbe0, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c42125b97c0c49f8b899aa9c82889b5c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/68dc8993850a4cb0983f8c45022fdf81, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/59ebba31bc4346968fc196e5d05a55e9] to archive 2024-12-04T20:30:38,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:30:38,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a34704895726432d987ffb4cea4436b9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a34704895726432d987ffb4cea4436b9 2024-12-04T20:30:38,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/47c830875d474bc48f53e7db022ae662 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/47c830875d474bc48f53e7db022ae662 2024-12-04T20:30:38,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/f6dbfa704f724cc8afad636010db8ea7 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/f6dbfa704f724cc8afad636010db8ea7 2024-12-04T20:30:38,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/beb02c381e214d9d8f6014360be16e33 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/beb02c381e214d9d8f6014360be16e33 2024-12-04T20:30:38,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a670be7bc72048f6975f103686d74599 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/a670be7bc72048f6975f103686d74599 2024-12-04T20:30:38,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/67f2dc3668e94e16bc19978e580323e6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/67f2dc3668e94e16bc19978e580323e6 2024-12-04T20:30:38,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/77ddeb6d392d473187120fa5e4f0ca19 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/77ddeb6d392d473187120fa5e4f0ca19 2024-12-04T20:30:38,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/92b4dee409af4ef886e62921895c2ffe to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/92b4dee409af4ef886e62921895c2ffe 2024-12-04T20:30:38,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/bb244f3e4da447618e9b67e7aab14b47 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/bb244f3e4da447618e9b67e7aab14b47 2024-12-04T20:30:38,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fd1042c1befe40869cdae69db17d4d04 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fd1042c1befe40869cdae69db17d4d04 2024-12-04T20:30:38,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/08ede8cb189340198b329c6181c90734 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/08ede8cb189340198b329c6181c90734 2024-12-04T20:30:38,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/53706001c81540cb891df874f44d49ca to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/53706001c81540cb891df874f44d49ca 2024-12-04T20:30:38,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2e5d3ff475e54409a4869b70d5f86749 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2e5d3ff475e54409a4869b70d5f86749 2024-12-04T20:30:38,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fdcb254776374452978d6fba5a5e7e92 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/fdcb254776374452978d6fba5a5e7e92 2024-12-04T20:30:38,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2d9c6874949d491ca4347a63b181bd5f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/2d9c6874949d491ca4347a63b181bd5f 2024-12-04T20:30:39,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/0e55b05d3e1844d3b7437ee7c5d74a76 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/0e55b05d3e1844d3b7437ee7c5d74a76 2024-12-04T20:30:39,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ff454c85aaff465eaa961ef1e8e5629b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ff454c85aaff465eaa961ef1e8e5629b 2024-12-04T20:30:39,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/97f00470c61c407097cdf2b8173ad7c2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/97f00470c61c407097cdf2b8173ad7c2 2024-12-04T20:30:39,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ba1cf72bd84c473dab33da6dca0acbe0 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/ba1cf72bd84c473dab33da6dca0acbe0 2024-12-04T20:30:39,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c42125b97c0c49f8b899aa9c82889b5c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c42125b97c0c49f8b899aa9c82889b5c 2024-12-04T20:30:39,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/68dc8993850a4cb0983f8c45022fdf81 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/68dc8993850a4cb0983f8c45022fdf81 2024-12-04T20:30:39,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/59ebba31bc4346968fc196e5d05a55e9 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/59ebba31bc4346968fc196e5d05a55e9 2024-12-04T20:30:39,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/973147bd46624cea8e3ece16e6cea4d5, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/81e9a9e9fe784483a0541357eaec9922, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7c67897da7244f8e9237ff33f9602655, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/1bb2804d13af4794906f53d06f93fe7f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4bc25cdb36c04b33b961a280150d8cf6, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6f9b9b8e452b4c48bf06c46606d02227, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf8ab3f8b47847588f5bb69baf0180eb, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/2f10d2d435a8423b8d91328fea5484d3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4ef6ac9540ac48f796778a036d5a6ba1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/79853bb3fe2d41c89425d8b830b0c8ae, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/585969f43ac743d6bb72556680621d93, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/9fe36b9620824932930bceac2c1cba33, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/cd3f1afd1b30443bafb6511bb0edd99f, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6cafe5f334db4a9da64782c66f59917d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/21ca4b3ded8d4549a7eee0ef6d325c09, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/f56e16064dd04b16bbb9f603bfa61a3a, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/425467c604dc46368c12d00f56f2ec93, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5bfe56de6dc14775a7086178a929b36c, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/016d8e615f7843c3975d4ce87987fcac, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7784053185984c9d96ee5a6af04f6900, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/51cf81af2a1c454790354c122ea8de27, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf474f96f32c415fbefa01d9d5c77759] to archive 2024-12-04T20:30:39,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:30:39,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/973147bd46624cea8e3ece16e6cea4d5 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/973147bd46624cea8e3ece16e6cea4d5 2024-12-04T20:30:39,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/81e9a9e9fe784483a0541357eaec9922 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/81e9a9e9fe784483a0541357eaec9922 2024-12-04T20:30:39,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7c67897da7244f8e9237ff33f9602655 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7c67897da7244f8e9237ff33f9602655 2024-12-04T20:30:39,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/1bb2804d13af4794906f53d06f93fe7f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/1bb2804d13af4794906f53d06f93fe7f 2024-12-04T20:30:39,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4bc25cdb36c04b33b961a280150d8cf6 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4bc25cdb36c04b33b961a280150d8cf6 2024-12-04T20:30:39,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6f9b9b8e452b4c48bf06c46606d02227 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6f9b9b8e452b4c48bf06c46606d02227 2024-12-04T20:30:39,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf8ab3f8b47847588f5bb69baf0180eb to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf8ab3f8b47847588f5bb69baf0180eb 2024-12-04T20:30:39,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/2f10d2d435a8423b8d91328fea5484d3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/2f10d2d435a8423b8d91328fea5484d3 2024-12-04T20:30:39,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4ef6ac9540ac48f796778a036d5a6ba1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/4ef6ac9540ac48f796778a036d5a6ba1 2024-12-04T20:30:39,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/79853bb3fe2d41c89425d8b830b0c8ae to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/79853bb3fe2d41c89425d8b830b0c8ae 2024-12-04T20:30:39,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/585969f43ac743d6bb72556680621d93 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/585969f43ac743d6bb72556680621d93 2024-12-04T20:30:39,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/9fe36b9620824932930bceac2c1cba33 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/9fe36b9620824932930bceac2c1cba33 2024-12-04T20:30:39,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/cd3f1afd1b30443bafb6511bb0edd99f to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/cd3f1afd1b30443bafb6511bb0edd99f 2024-12-04T20:30:39,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6cafe5f334db4a9da64782c66f59917d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/6cafe5f334db4a9da64782c66f59917d 2024-12-04T20:30:39,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/21ca4b3ded8d4549a7eee0ef6d325c09 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/21ca4b3ded8d4549a7eee0ef6d325c09 2024-12-04T20:30:39,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/f56e16064dd04b16bbb9f603bfa61a3a to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/f56e16064dd04b16bbb9f603bfa61a3a 2024-12-04T20:30:39,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/425467c604dc46368c12d00f56f2ec93 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/425467c604dc46368c12d00f56f2ec93 2024-12-04T20:30:39,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5bfe56de6dc14775a7086178a929b36c to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5bfe56de6dc14775a7086178a929b36c 2024-12-04T20:30:39,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/016d8e615f7843c3975d4ce87987fcac to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/016d8e615f7843c3975d4ce87987fcac 2024-12-04T20:30:39,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7784053185984c9d96ee5a6af04f6900 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/7784053185984c9d96ee5a6af04f6900 2024-12-04T20:30:39,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/51cf81af2a1c454790354c122ea8de27 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/51cf81af2a1c454790354c122ea8de27 2024-12-04T20:30:39,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf474f96f32c415fbefa01d9d5c77759 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/bf474f96f32c415fbefa01d9d5c77759 2024-12-04T20:30:39,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/e55acd0f6a9d435f83f89f190c4ef90e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9abf45f7566345b196229fc3c42d54f4, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8ef63d51dbb3424ea1284c4f07052b9d, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/df060a94dc254e4e87c24688cfa9dc46, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c3e2a0d2ef3b476792f806207b229d39, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/890d2b5debcc4b28ac4e99da34ac8502, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/64809aafb47f4ab8867a7e517b59fbc1, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9d5fa5c7ab434f718c4848470f2df6f2, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/b3f7339fe8a54a34973f146c0d08c5df, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0d0a1e978d3449290aef3930b245861, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8e52feed69ce4348a32dfb3a20dd194e, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/87ee3c7555a942f6a5e7214532b58f2b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/23be68490be34b4887e4b6747276d4ca, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/d4357fb8f6bd4c1890dcc52279da3ff8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0354a5fae3544aba253f102653ac1f3, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/dab322fbc2434adc872181ec711fb9e8, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6060b720bfd344fdbb8148db639754fe, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c766a2b611dd41f398f43dca57335096, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/0641819ea9f14d379e811c6b9296d1ec, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/59502ecb5e274126a788650af288d95b, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/08f770a0896b4fc0b9d379f7c9a86d04, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/3ee2f34ffbb540aba05d2bcbc8010205] to archive 2024-12-04T20:30:39,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:30:39,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/e55acd0f6a9d435f83f89f190c4ef90e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/e55acd0f6a9d435f83f89f190c4ef90e 2024-12-04T20:30:39,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9abf45f7566345b196229fc3c42d54f4 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9abf45f7566345b196229fc3c42d54f4 2024-12-04T20:30:39,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8ef63d51dbb3424ea1284c4f07052b9d to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8ef63d51dbb3424ea1284c4f07052b9d 2024-12-04T20:30:39,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/df060a94dc254e4e87c24688cfa9dc46 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/df060a94dc254e4e87c24688cfa9dc46 2024-12-04T20:30:39,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c3e2a0d2ef3b476792f806207b229d39 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c3e2a0d2ef3b476792f806207b229d39 2024-12-04T20:30:39,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/890d2b5debcc4b28ac4e99da34ac8502 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/890d2b5debcc4b28ac4e99da34ac8502 2024-12-04T20:30:39,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/64809aafb47f4ab8867a7e517b59fbc1 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/64809aafb47f4ab8867a7e517b59fbc1 2024-12-04T20:30:39,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9d5fa5c7ab434f718c4848470f2df6f2 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/9d5fa5c7ab434f718c4848470f2df6f2 2024-12-04T20:30:39,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/b3f7339fe8a54a34973f146c0d08c5df to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/b3f7339fe8a54a34973f146c0d08c5df 2024-12-04T20:30:39,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0d0a1e978d3449290aef3930b245861 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0d0a1e978d3449290aef3930b245861 2024-12-04T20:30:39,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8e52feed69ce4348a32dfb3a20dd194e to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/8e52feed69ce4348a32dfb3a20dd194e 2024-12-04T20:30:39,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/87ee3c7555a942f6a5e7214532b58f2b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/87ee3c7555a942f6a5e7214532b58f2b 2024-12-04T20:30:39,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/23be68490be34b4887e4b6747276d4ca to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/23be68490be34b4887e4b6747276d4ca 2024-12-04T20:30:39,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/d4357fb8f6bd4c1890dcc52279da3ff8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/d4357fb8f6bd4c1890dcc52279da3ff8 2024-12-04T20:30:39,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0354a5fae3544aba253f102653ac1f3 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/f0354a5fae3544aba253f102653ac1f3 2024-12-04T20:30:39,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/dab322fbc2434adc872181ec711fb9e8 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/dab322fbc2434adc872181ec711fb9e8 2024-12-04T20:30:39,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6060b720bfd344fdbb8148db639754fe to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6060b720bfd344fdbb8148db639754fe 2024-12-04T20:30:39,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c766a2b611dd41f398f43dca57335096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/c766a2b611dd41f398f43dca57335096 2024-12-04T20:30:39,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/0641819ea9f14d379e811c6b9296d1ec to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/0641819ea9f14d379e811c6b9296d1ec 2024-12-04T20:30:39,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/59502ecb5e274126a788650af288d95b to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/59502ecb5e274126a788650af288d95b 2024-12-04T20:30:39,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/08f770a0896b4fc0b9d379f7c9a86d04 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/08f770a0896b4fc0b9d379f7c9a86d04 2024-12-04T20:30:39,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/3ee2f34ffbb540aba05d2bcbc8010205 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/3ee2f34ffbb540aba05d2bcbc8010205 2024-12-04T20:30:39,045 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/recovered.edits/329.seqid, newMaxSeqId=329, maxSeqId=4 2024-12-04T20:30:39,045 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. 2024-12-04T20:30:39,045 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for eec55403d95e6b84b5f2b496767f2096: Waiting for close lock at 1733344237329Running coprocessor pre-close hooks at 1733344237329Disabling compacts and flushes for region at 1733344237329Disabling writes for close at 1733344237329Obtaining lock to block concurrent updates at 1733344237329Preparing flush snapshotting stores in eec55403d95e6b84b5f2b496767f2096 at 1733344237329Finished memstore snapshotting TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733344237330 (+1 ms)Flushing stores of TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096. at 1733344237330Flushing eec55403d95e6b84b5f2b496767f2096/A: creating writer at 1733344237330Flushing mob file eec55403d95e6b84b5f2b496767f2096/A: appending metadata at 1733344237334 (+4 ms)Flushing mob file eec55403d95e6b84b5f2b496767f2096/A: closing flushed file at 1733344237334Flushing eec55403d95e6b84b5f2b496767f2096/A: appending metadata at 1733344237748 (+414 ms)Flushing eec55403d95e6b84b5f2b496767f2096/A: closing flushed file at 1733344237748Flushing eec55403d95e6b84b5f2b496767f2096/B: creating writer at 1733344238156 (+408 ms)Flushing eec55403d95e6b84b5f2b496767f2096/B: appending metadata at 1733344238158 (+2 ms)Flushing eec55403d95e6b84b5f2b496767f2096/B: closing flushed file at 1733344238158Flushing eec55403d95e6b84b5f2b496767f2096/C: creating writer at 1733344238566 (+408 ms)Flushing eec55403d95e6b84b5f2b496767f2096/C: appending metadata at 1733344238568 (+2 ms)Flushing eec55403d95e6b84b5f2b496767f2096/C: closing flushed file at 1733344238568Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e8a1cd6: reopening flushed file at 1733344238975 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c87d2b8: reopening flushed file at 1733344238978 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47f53609: reopening flushed file at 1733344238981 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for eec55403d95e6b84b5f2b496767f2096 in 1656ms, sequenceid=326, compaction requested=false at 1733344238985 (+4 ms)Writing region close event to WAL at 1733344239042 (+57 ms)Running coprocessor post-close hooks at 1733344239045 (+3 ms)Closed at 1733344239045 2024-12-04T20:30:39,046 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,047 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=eec55403d95e6b84b5f2b496767f2096, regionState=CLOSED 2024-12-04T20:30:39,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 because future has completed 2024-12-04T20:30:39,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-04T20:30:39,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure eec55403d95e6b84b5f2b496767f2096, server=c2ef38372881,45443,1733344054291 in 1.8710 sec 2024-12-04T20:30:39,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-04T20:30:39,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eec55403d95e6b84b5f2b496767f2096, UNASSIGN in 1.8750 sec 2024-12-04T20:30:39,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-04T20:30:39,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-12-04T20:30:39,054 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733344239053"}]},"ts":"1733344239053"} 2024-12-04T20:30:39,055 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T20:30:39,055 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T20:30:39,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8870 sec 2024-12-04T20:30:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T20:30:39,322 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:39,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T20:30:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:39,326 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T20:30:39,328 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:39,331 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,333 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C, FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/recovered.edits] 2024-12-04T20:30:39,334 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/4b912415444542ecab3f869595390330 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/4b912415444542ecab3f869595390330 2024-12-04T20:30:39,335 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c9525d709d7e487b8eaa0dce429ef115 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/A/c9525d709d7e487b8eaa0dce429ef115 2024-12-04T20:30:39,337 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5c510117a1a54eb59ec685f9c1514035 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/5c510117a1a54eb59ec685f9c1514035 2024-12-04T20:30:39,337 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/c3c7635022e6463f9fc842e38d0f6301 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/B/c3c7635022e6463f9fc842e38d0f6301 2024-12-04T20:30:39,339 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6fa109cc39484809a5e60602d91cf466 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/6fa109cc39484809a5e60602d91cf466 2024-12-04T20:30:39,339 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/aa50b80899e9424b952f741a2bb10a34 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/C/aa50b80899e9424b952f741a2bb10a34 2024-12-04T20:30:39,341 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/recovered.edits/329.seqid to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096/recovered.edits/329.seqid 2024-12-04T20:30:39,341 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/default/TestAcidGuarantees/eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T20:30:39,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T20:30:39,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T20:30:39,344 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042494b0f03dff4ffa97686141bca0072c_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042494b0f03dff4ffa97686141bca0072c_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,344 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042f8a30190a9d440e85af70d85a4bff5d_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042f8a30190a9d440e85af70d85a4bff5d_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,345 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204371d4fd794aa48d0ab8bab58586749b0_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204371d4fd794aa48d0ab8bab58586749b0_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,346 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045a65b5d394ee46d2b7247ae9679b0f14_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045a65b5d394ee46d2b7247ae9679b0f14_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,347 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045cf4bad543e24d698d83a627649727b2_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045cf4bad543e24d698d83a627649727b2_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,348 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120466149cfadee74b09bbac38d8bda405c5_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120466149cfadee74b09bbac38d8bda405c5_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,354 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120485776ee7be414130acbd654f21a68822_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120485776ee7be414130acbd654f21a68822_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,355 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a37a7918785245648acdef09749997a6_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a37a7918785245648acdef09749997a6_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,356 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a7bcfb87fa1f45ad93ff7b8469e9475d_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a7bcfb87fa1f45ad93ff7b8469e9475d_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,357 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ac2ab56aeffc4dfe92a44286017cb180_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ac2ab56aeffc4dfe92a44286017cb180_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,359 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b13cfde5729c4373bb04e1b7d9f9355a_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b13cfde5729c4373bb04e1b7d9f9355a_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,360 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b8a4bedb3330458e86d9f8899ad12925_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b8a4bedb3330458e86d9f8899ad12925_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,361 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c2633ed00f404d2f96ce49650d28e0df_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c2633ed00f404d2f96ce49650d28e0df_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,362 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cf818dd89b594e1fbaaa7a56a60aef41_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cf818dd89b594e1fbaaa7a56a60aef41_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,363 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d6000b1b218043c9a4f94669a47da364_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d6000b1b218043c9a4f94669a47da364_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,364 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e2c7458fa499462c9319c4eabddd5a42_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e2c7458fa499462c9319c4eabddd5a42_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,365 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e6e2c61fb02044fdbd3b04eb2365833e_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e6e2c61fb02044fdbd3b04eb2365833e_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,366 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204eeb690563d204c1e98cc5e2b504961dd_eec55403d95e6b84b5f2b496767f2096 to hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204eeb690563d204c1e98cc5e2b504961dd_eec55403d95e6b84b5f2b496767f2096 2024-12-04T20:30:39,366 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T20:30:39,369 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:39,371 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T20:30:39,373 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T20:30:39,374 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:39,374 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T20:30:39,374 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733344239374"}]},"ts":"9223372036854775807"} 2024-12-04T20:30:39,376 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T20:30:39,376 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eec55403d95e6b84b5f2b496767f2096, NAME => 'TestAcidGuarantees,,1733344205547.eec55403d95e6b84b5f2b496767f2096.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T20:30:39,376 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T20:30:39,376 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733344239376"}]},"ts":"9223372036854775807"} 2024-12-04T20:30:39,378 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T20:30:39,378 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T20:30:39,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-12-04T20:30:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T20:30:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T20:30:39,436 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T20:30:39,443 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 251), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=518 (was 491) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4592 (was 4342) - AvailableMemoryMB LEAK? - 2024-12-04T20:30:39,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:30:39,444 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:30:39,444 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:30:39,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:39,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:39,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:30:39,444 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:30:39,444 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=974344401, stopped=false 2024-12-04T20:30:39,445 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,43507,1733344053664 2024-12-04T20:30:39,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:30:39,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:30:39,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:30:39,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:30:39,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:30:39,446 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:30:39,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:30:39,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:39,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:30:39,447 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,45443,1733344054291' ***** 2024-12-04T20:30:39,447 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:30:39,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:30:39,447 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:30:39,448 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,45443,1733344054291 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:45443. 2024-12-04T20:30:39,448 DEBUG [RS:0;c2ef38372881:45443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:30:39,448 DEBUG [RS:0;c2ef38372881:45443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:30:39,448 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:30:39,449 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T20:30:39,449 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T20:30:39,449 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:39,449 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:30:39,449 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:30:39,449 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:30:39,449 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:30:39,450 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:30:39,450 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-04T20:30:39,472 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/info/20968ecf092643198b6410620af8afdd is 95, key is TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc./info:/1733344082026/DeleteFamily/seqid=0 2024-12-04T20:30:39,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742459_1635 (size=6514) 2024-12-04T20:30:39,517 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:30:39,650 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:39,850 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:39,876 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/info/20968ecf092643198b6410620af8afdd 2024-12-04T20:30:39,900 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/ns/a102e18f06b24903976d3162664c1f0b is 93, key is TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc./ns:/1733344081975/DeleteFamily/seqid=0 2024-12-04T20:30:39,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742460_1636 (size=5906) 2024-12-04T20:30:40,050 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:40,250 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:40,267 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T20:30:40,267 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T20:30:40,304 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/ns/a102e18f06b24903976d3162664c1f0b 2024-12-04T20:30:40,328 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/rep_barrier/3ac74ab050de460a9b0a2b45f09bacb9 is 102, key is TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc./rep_barrier:/1733344081975/DeleteFamily/seqid=0 2024-12-04T20:30:40,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742461_1637 (size=6025) 2024-12-04T20:30:40,451 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:40,651 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:40,732 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/rep_barrier/3ac74ab050de460a9b0a2b45f09bacb9 2024-12-04T20:30:40,757 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/table/512d6292dc454dd3bb6ab5a6d7e3e8d1 is 96, key is TestAcidGuarantees,,1733344057715.e4d991821bf5842b6ec138d6f2a184fc./table:/1733344081975/DeleteFamily/seqid=0 2024-12-04T20:30:40,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742462_1638 (size=5936) 2024-12-04T20:30:40,851 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:41,051 DEBUG [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:30:41,161 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/table/512d6292dc454dd3bb6ab5a6d7e3e8d1 2024-12-04T20:30:41,165 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/info/20968ecf092643198b6410620af8afdd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/info/20968ecf092643198b6410620af8afdd 2024-12-04T20:30:41,169 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/info/20968ecf092643198b6410620af8afdd, entries=12, sequenceid=90, filesize=6.4 K 2024-12-04T20:30:41,170 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/ns/a102e18f06b24903976d3162664c1f0b as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/ns/a102e18f06b24903976d3162664c1f0b 2024-12-04T20:30:41,174 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/ns/a102e18f06b24903976d3162664c1f0b, entries=8, sequenceid=90, filesize=5.8 K 2024-12-04T20:30:41,175 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/rep_barrier/3ac74ab050de460a9b0a2b45f09bacb9 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/rep_barrier/3ac74ab050de460a9b0a2b45f09bacb9 2024-12-04T20:30:41,178 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/rep_barrier/3ac74ab050de460a9b0a2b45f09bacb9, entries=6, sequenceid=90, filesize=5.9 K 2024-12-04T20:30:41,179 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/.tmp/table/512d6292dc454dd3bb6ab5a6d7e3e8d1 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/table/512d6292dc454dd3bb6ab5a6d7e3e8d1 2024-12-04T20:30:41,182 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/table/512d6292dc454dd3bb6ab5a6d7e3e8d1, entries=7, sequenceid=90, filesize=5.8 K 2024-12-04T20:30:41,182 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false 2024-12-04T20:30:41,186 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-04T20:30:41,187 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:30:41,187 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:30:41,187 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733344239449Running coprocessor pre-close hooks at 1733344239449Disabling compacts and flushes for region at 1733344239449Disabling writes for close at 1733344239450 (+1 ms)Obtaining lock to block concurrent updates at 1733344239450Preparing flush snapshotting stores in 1588230740 at 1733344239450Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733344239450Flushing stores of hbase:meta,,1.1588230740 at 1733344239451 (+1 ms)Flushing 1588230740/info: creating writer at 1733344239451Flushing 1588230740/info: appending metadata at 1733344239471 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733344239471Flushing 1588230740/ns: creating writer at 1733344239885 (+414 ms)Flushing 1588230740/ns: appending metadata at 1733344239900 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733344239900Flushing 1588230740/rep_barrier: creating writer at 1733344240312 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733344240327 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733344240327Flushing 1588230740/table: creating writer at 1733344240741 (+414 ms)Flushing 1588230740/table: appending metadata at 1733344240756 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733344240756Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e65e5a0: reopening flushed file at 1733344241165 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29612f81: reopening flushed file at 1733344241169 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66464272: reopening flushed file at 1733344241174 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37e1445: reopening flushed file at 1733344241178 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false at 1733344241183 (+5 ms)Writing region close event to WAL at 1733344241184 (+1 ms)Running coprocessor post-close hooks at 1733344241187 (+3 ms)Closed at 1733344241187 2024-12-04T20:30:41,187 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:30:41,252 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,45443,1733344054291; all regions closed. 2024-12-04T20:30:41,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741834_1010 (size=25344) 2024-12-04T20:30:41,263 DEBUG [RS:0;c2ef38372881:45443 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/oldWALs 2024-12-04T20:30:41,264 INFO [RS:0;c2ef38372881:45443 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2ef38372881%2C45443%2C1733344054291.meta:.meta(num 1733344057290) 2024-12-04T20:30:41,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741832_1008 (size=20631514) 2024-12-04T20:30:41,270 DEBUG [RS:0;c2ef38372881:45443 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/oldWALs 2024-12-04T20:30:41,270 INFO [RS:0;c2ef38372881:45443 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2ef38372881%2C45443%2C1733344054291:(num 1733344056326) 2024-12-04T20:30:41,270 DEBUG [RS:0;c2ef38372881:45443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:30:41,270 INFO [RS:0;c2ef38372881:45443 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:30:41,270 INFO [RS:0;c2ef38372881:45443 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:30:41,270 INFO [RS:0;c2ef38372881:45443 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T20:30:41,271 INFO [RS:0;c2ef38372881:45443 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:30:41,271 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:30:41,272 INFO [RS:0;c2ef38372881:45443 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45443 2024-12-04T20:30:41,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,45443,1733344054291 2024-12-04T20:30:41,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:30:41,275 INFO [RS:0;c2ef38372881:45443 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:30:41,277 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,45443,1733344054291] 2024-12-04T20:30:41,278 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,45443,1733344054291 already deleted, retry=false 2024-12-04T20:30:41,279 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,45443,1733344054291 expired; onlineServers=0 2024-12-04T20:30:41,279 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,43507,1733344053664' ***** 2024-12-04T20:30:41,279 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:30:41,279 INFO [M:0;c2ef38372881:43507 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:30:41,279 INFO [M:0;c2ef38372881:43507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:30:41,279 DEBUG [M:0;c2ef38372881:43507 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:30:41,279 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:30:41,279 DEBUG [M:0;c2ef38372881:43507 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:30:41,279 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733344056050 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733344056050,5,FailOnTimeoutGroup] 2024-12-04T20:30:41,279 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733344056045 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733344056045,5,FailOnTimeoutGroup] 2024-12-04T20:30:41,280 INFO [M:0;c2ef38372881:43507 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:30:41,280 INFO [M:0;c2ef38372881:43507 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:30:41,280 DEBUG [M:0;c2ef38372881:43507 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:30:41,280 INFO [M:0;c2ef38372881:43507 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:30:41,280 INFO [M:0;c2ef38372881:43507 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:30:41,280 ERROR [M:0;c2ef38372881:43507 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-04T20:30:41,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:30:41,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:30:41,281 INFO [M:0;c2ef38372881:43507 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:30:41,281 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:30:41,282 DEBUG [M:0;c2ef38372881:43507 {}] zookeeper.ZKUtil(347): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:30:41,282 WARN [M:0;c2ef38372881:43507 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:30:41,282 INFO [M:0;c2ef38372881:43507 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/.lastflushedseqids 2024-12-04T20:30:41,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742463_1639 (size=72) 2024-12-04T20:30:41,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:30:41,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45443-0x100a6eaa2e80001, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:30:41,378 INFO [RS:0;c2ef38372881:45443 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:30:41,378 INFO [RS:0;c2ef38372881:45443 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,45443,1733344054291; zookeeper connection closed. 2024-12-04T20:30:41,380 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@43ef3132 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@43ef3132 2024-12-04T20:30:41,381 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T20:30:41,691 INFO [M:0;c2ef38372881:43507 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:30:41,691 INFO [M:0;c2ef38372881:43507 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:30:41,691 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:30:41,691 INFO [M:0;c2ef38372881:43507 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:30:41,691 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:30:41,691 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:30:41,691 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:30:41,692 INFO [M:0;c2ef38372881:43507 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=759.58 KB heapSize=929.84 KB 2024-12-04T20:30:41,706 DEBUG [M:0;c2ef38372881:43507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e5801b38a66c408fb418d2bdc397debd is 82, key is hbase:meta,,1/info:regioninfo/1733344057376/Put/seqid=0 2024-12-04T20:30:41,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742464_1640 (size=5672) 2024-12-04T20:30:42,110 INFO [M:0;c2ef38372881:43507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2098 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e5801b38a66c408fb418d2bdc397debd 2024-12-04T20:30:42,138 DEBUG [M:0;c2ef38372881:43507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a16a578d2f2d4767894df1171c625443 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1733344208654/Put/seqid=0 2024-12-04T20:30:42,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742465_1641 (size=44644) 2024-12-04T20:30:42,542 INFO [M:0;c2ef38372881:43507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=759.02 KB at sequenceid=2098 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a16a578d2f2d4767894df1171c625443 2024-12-04T20:30:42,547 INFO [M:0;c2ef38372881:43507 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a16a578d2f2d4767894df1171c625443 2024-12-04T20:30:42,566 DEBUG [M:0;c2ef38372881:43507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a783ed9df88d4c24b8191bc397bc9416 is 69, key is c2ef38372881,45443,1733344054291/rs:state/1733344056094/Put/seqid=0 2024-12-04T20:30:42,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073742466_1642 (size=5156) 2024-12-04T20:30:42,970 INFO [M:0;c2ef38372881:43507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2098 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a783ed9df88d4c24b8191bc397bc9416 2024-12-04T20:30:42,972 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e5801b38a66c408fb418d2bdc397debd as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e5801b38a66c408fb418d2bdc397debd 2024-12-04T20:30:42,975 INFO [M:0;c2ef38372881:43507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e5801b38a66c408fb418d2bdc397debd, entries=8, sequenceid=2098, filesize=5.5 K 2024-12-04T20:30:42,976 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a16a578d2f2d4767894df1171c625443 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a16a578d2f2d4767894df1171c625443 2024-12-04T20:30:42,979 INFO [M:0;c2ef38372881:43507 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a16a578d2f2d4767894df1171c625443 2024-12-04T20:30:42,979 INFO [M:0;c2ef38372881:43507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a16a578d2f2d4767894df1171c625443, entries=166, sequenceid=2098, filesize=43.6 K 2024-12-04T20:30:42,979 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a783ed9df88d4c24b8191bc397bc9416 as hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a783ed9df88d4c24b8191bc397bc9416 2024-12-04T20:30:42,982 INFO [M:0;c2ef38372881:43507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/fdda03e7-3855-d6d7-be9d-5ff6c6b1ffc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a783ed9df88d4c24b8191bc397bc9416, entries=1, sequenceid=2098, filesize=5.0 K 2024-12-04T20:30:42,983 INFO [M:0;c2ef38372881:43507 {}] regionserver.HRegion(3140): Finished flush of dataSize ~759.58 KB/777805, heapSize ~929.54 KB/951848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1292ms, sequenceid=2098, compaction requested=false 2024-12-04T20:30:42,984 INFO [M:0;c2ef38372881:43507 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:30:42,984 DEBUG [M:0;c2ef38372881:43507 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733344241691Disabling compacts and flushes for region at 1733344241691Disabling writes for close at 1733344241691Obtaining lock to block concurrent updates at 1733344241692 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733344241692Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=777805, getHeapSize=952088, getOffHeapSize=0, getCellsCount=2289 at 1733344241692Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733344241692Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733344241692Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733344241706 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733344241706Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733344242118 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733344242138 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733344242138Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733344242547 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733344242566 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733344242566Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b8ade81: reopening flushed file at 1733344242972 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b26300b: reopening flushed file at 1733344242975 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3969b6a8: reopening flushed file at 1733344242979 (+4 ms)Finished flush of dataSize ~759.58 KB/777805, heapSize ~929.54 KB/951848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1292ms, sequenceid=2098, compaction requested=false at 1733344242983 (+4 ms)Writing region close event to WAL at 1733344242984 (+1 ms)Closed at 1733344242984 2024-12-04T20:30:42,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42649 is added to blk_1073741830_1006 (size=915358) 2024-12-04T20:30:42,987 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:30:42,987 INFO [M:0;c2ef38372881:43507 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:30:42,987 INFO [M:0;c2ef38372881:43507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43507 2024-12-04T20:30:42,987 INFO [M:0;c2ef38372881:43507 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:30:43,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:30:43,089 INFO [M:0;c2ef38372881:43507 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:30:43,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43507-0x100a6eaa2e80000, quorum=127.0.0.1:57628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:30:43,094 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:30:43,097 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:30:43,097 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:30:43,097 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:30:43,097 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/hadoop.log.dir/,STOPPED} 2024-12-04T20:30:43,100 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:30:43,100 WARN [BP-994321494-172.17.0.2-1733344050940 heartbeating to localhost/127.0.0.1:41363 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:30:43,100 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:30:43,100 WARN [BP-994321494-172.17.0.2-1733344050940 heartbeating to localhost/127.0.0.1:41363 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-994321494-172.17.0.2-1733344050940 (Datanode Uuid 4ea7031d-452d-42d8-9831-975d4a4a5e30) service to localhost/127.0.0.1:41363 2024-12-04T20:30:43,102 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4/data/data1/current/BP-994321494-172.17.0.2-1733344050940 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:30:43,102 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/cluster_321cdca5-9dcb-8b7b-bca6-ef54e943d2f4/data/data2/current/BP-994321494-172.17.0.2-1733344050940 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:30:43,103 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:30:43,112 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:30:43,113 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:30:43,113 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:30:43,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:30:43,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cb6fd566-7708-3d3c-0bbf-9ff5f7f82983/hadoop.log.dir/,STOPPED} 2024-12-04T20:30:43,126 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:30:43,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down